2024-11-18 20:11:52,559 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-11-18 20:11:52,576 main DEBUG Took 0.014470 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-18 20:11:52,577 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-18 20:11:52,577 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-18 20:11:52,578 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-18 20:11:52,580 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-18 20:11:52,590 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-18 20:11:52,608 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-18 20:11:52,618 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-18 20:11:52,619 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-18 20:11:52,619 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-18 20:11:52,620 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-18 20:11:52,621 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-18 20:11:52,622 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-18 20:11:52,622 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-18 20:11:52,623 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-18 20:11:52,623 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-18 20:11:52,625 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-18 20:11:52,625 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-18 20:11:52,626 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-18 20:11:52,627 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-18 20:11:52,627 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-18 20:11:52,628 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-18 20:11:52,629 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-18 20:11:52,629 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-18 20:11:52,630 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-18 20:11:52,631 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-18 20:11:52,631 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-18 20:11:52,632 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-18 20:11:52,633 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-18 20:11:52,633 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-18 20:11:52,634 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-18 20:11:52,634 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-18 20:11:52,636 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-18 20:11:52,638 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-18 20:11:52,640 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-18 20:11:52,640 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-18 20:11:52,642 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-18 20:11:52,643 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-18 20:11:52,653 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-18 20:11:52,657 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-18 20:11:52,659 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-18 20:11:52,660 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-18 20:11:52,660 main DEBUG createAppenders(={Console}) 2024-11-18 20:11:52,661 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad initialized 2024-11-18 20:11:52,662 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-11-18 20:11:52,662 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad OK. 2024-11-18 20:11:52,663 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-18 20:11:52,663 main DEBUG OutputStream closed 2024-11-18 20:11:52,664 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-18 20:11:52,664 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-18 20:11:52,664 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@55634720 OK 2024-11-18 20:11:52,790 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-18 20:11:52,794 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-18 20:11:52,796 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-18 20:11:52,797 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-18 20:11:52,798 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-18 20:11:52,799 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-18 20:11:52,800 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-18 20:11:52,800 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-18 20:11:52,801 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-18 20:11:52,801 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-18 20:11:52,802 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-18 20:11:52,802 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-18 20:11:52,803 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-18 20:11:52,803 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-18 20:11:52,804 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-18 20:11:52,804 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-18 20:11:52,805 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-18 20:11:52,806 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-18 20:11:52,809 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-18 20:11:52,810 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-logging/target/hbase-logging-3.0.0-beta-2-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@19835e64) with optional ClassLoader: null 2024-11-18 20:11:52,811 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-18 20:11:52,813 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@19835e64] started OK. 2024-11-18T20:11:53,241 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/098aacef-40a4-13ba-ed62-ee93d3ee3293 2024-11-18 20:11:53,245 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-18 20:11:53,245 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-18T20:11:53,257 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.TestAcidGuaranteesWithNoInMemCompaction timeout: 13 mins 2024-11-18T20:11:53,320 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=1, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-18T20:11:53,350 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/098aacef-40a4-13ba-ed62-ee93d3ee3293/cluster_08cd2a1d-a16d-5ba4-ed14-6adb17e4da61, deleteOnExit=true 2024-11-18T20:11:53,350 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-11-18T20:11:53,352 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/098aacef-40a4-13ba-ed62-ee93d3ee3293/test.cache.data in system properties and HBase conf 2024-11-18T20:11:53,353 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/098aacef-40a4-13ba-ed62-ee93d3ee3293/hadoop.tmp.dir in system properties and HBase conf 2024-11-18T20:11:53,356 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/098aacef-40a4-13ba-ed62-ee93d3ee3293/hadoop.log.dir in system properties and HBase conf 2024-11-18T20:11:53,357 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/098aacef-40a4-13ba-ed62-ee93d3ee3293/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-18T20:11:53,358 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/098aacef-40a4-13ba-ed62-ee93d3ee3293/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-18T20:11:53,358 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-18T20:11:53,476 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-18T20:11:53,604 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-18T20:11:53,610 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/098aacef-40a4-13ba-ed62-ee93d3ee3293/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-18T20:11:53,611 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/098aacef-40a4-13ba-ed62-ee93d3ee3293/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-18T20:11:53,612 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/098aacef-40a4-13ba-ed62-ee93d3ee3293/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-18T20:11:53,612 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/098aacef-40a4-13ba-ed62-ee93d3ee3293/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-18T20:11:53,613 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/098aacef-40a4-13ba-ed62-ee93d3ee3293/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-18T20:11:53,614 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/098aacef-40a4-13ba-ed62-ee93d3ee3293/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-18T20:11:53,614 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/098aacef-40a4-13ba-ed62-ee93d3ee3293/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-18T20:11:53,615 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/098aacef-40a4-13ba-ed62-ee93d3ee3293/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-18T20:11:53,616 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/098aacef-40a4-13ba-ed62-ee93d3ee3293/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-18T20:11:53,616 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/098aacef-40a4-13ba-ed62-ee93d3ee3293/nfs.dump.dir in system properties and HBase conf 2024-11-18T20:11:53,617 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/098aacef-40a4-13ba-ed62-ee93d3ee3293/java.io.tmpdir in system properties and HBase conf 2024-11-18T20:11:53,617 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/098aacef-40a4-13ba-ed62-ee93d3ee3293/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-18T20:11:53,618 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/098aacef-40a4-13ba-ed62-ee93d3ee3293/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-18T20:11:53,619 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/098aacef-40a4-13ba-ed62-ee93d3ee3293/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-18T20:11:54,532 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-11-18T20:11:54,618 INFO [Time-limited test {}] log.Log(170): Logging initialized @3024ms to org.eclipse.jetty.util.log.Slf4jLog 2024-11-18T20:11:54,727 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-18T20:11:54,819 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-18T20:11:54,858 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-18T20:11:54,858 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-18T20:11:54,860 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-18T20:11:54,881 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-18T20:11:54,887 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6b23711f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/098aacef-40a4-13ba-ed62-ee93d3ee3293/hadoop.log.dir/,AVAILABLE} 2024-11-18T20:11:54,889 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@93c62df{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-18T20:11:55,111 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4ded99d3{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/098aacef-40a4-13ba-ed62-ee93d3ee3293/java.io.tmpdir/jetty-localhost-37459-hadoop-hdfs-3_4_1-tests_jar-_-any-15774112735941838347/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-18T20:11:55,123 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@557285fa{HTTP/1.1, (http/1.1)}{localhost:37459} 2024-11-18T20:11:55,124 INFO [Time-limited test {}] server.Server(415): Started @3530ms 2024-11-18T20:11:55,541 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-18T20:11:55,550 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-18T20:11:55,552 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-18T20:11:55,552 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-18T20:11:55,552 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-18T20:11:55,553 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@336e9627{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/098aacef-40a4-13ba-ed62-ee93d3ee3293/hadoop.log.dir/,AVAILABLE} 2024-11-18T20:11:55,554 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@42946cd0{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-18T20:11:55,663 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@2c68b1bf{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/098aacef-40a4-13ba-ed62-ee93d3ee3293/java.io.tmpdir/jetty-localhost-33963-hadoop-hdfs-3_4_1-tests_jar-_-any-724408330593354947/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-18T20:11:55,664 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@3efe07f8{HTTP/1.1, (http/1.1)}{localhost:33963} 2024-11-18T20:11:55,665 INFO [Time-limited test {}] server.Server(415): Started @4071ms 2024-11-18T20:11:55,723 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-18T20:11:56,333 WARN [Thread-72 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/098aacef-40a4-13ba-ed62-ee93d3ee3293/cluster_08cd2a1d-a16d-5ba4-ed14-6adb17e4da61/data/data1/current/BP-138538035-172.17.0.2-1731960714281/current, will proceed with Du for space computation calculation, 2024-11-18T20:11:56,334 WARN [Thread-73 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/098aacef-40a4-13ba-ed62-ee93d3ee3293/cluster_08cd2a1d-a16d-5ba4-ed14-6adb17e4da61/data/data2/current/BP-138538035-172.17.0.2-1731960714281/current, will proceed with Du for space computation calculation, 2024-11-18T20:11:56,428 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-18T20:11:56,489 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x80d570fb32d90003 with lease ID 0x7188e03f06ba1384: Processing first storage report for DS-d929842b-b907-444e-b62d-3ef7011a945c from datanode DatanodeRegistration(127.0.0.1:41017, datanodeUuid=f25a728a-67d8-468a-a294-678d073a2b85, infoPort=40361, infoSecurePort=0, ipcPort=34575, storageInfo=lv=-57;cid=testClusterID;nsid=855321071;c=1731960714281) 2024-11-18T20:11:56,491 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x80d570fb32d90003 with lease ID 0x7188e03f06ba1384: from storage DS-d929842b-b907-444e-b62d-3ef7011a945c node DatanodeRegistration(127.0.0.1:41017, datanodeUuid=f25a728a-67d8-468a-a294-678d073a2b85, infoPort=40361, infoSecurePort=0, ipcPort=34575, storageInfo=lv=-57;cid=testClusterID;nsid=855321071;c=1731960714281), blocks: 0, hasStaleStorage: true, processing time: 3 msecs, invalidatedBlocks: 0 2024-11-18T20:11:56,492 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x80d570fb32d90003 with lease ID 0x7188e03f06ba1384: Processing first storage report for DS-50475cf3-4f38-44e4-97c8-ad4373908856 from datanode DatanodeRegistration(127.0.0.1:41017, datanodeUuid=f25a728a-67d8-468a-a294-678d073a2b85, infoPort=40361, infoSecurePort=0, ipcPort=34575, storageInfo=lv=-57;cid=testClusterID;nsid=855321071;c=1731960714281) 2024-11-18T20:11:56,492 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x80d570fb32d90003 with lease ID 0x7188e03f06ba1384: from storage DS-50475cf3-4f38-44e4-97c8-ad4373908856 node DatanodeRegistration(127.0.0.1:41017, datanodeUuid=f25a728a-67d8-468a-a294-678d073a2b85, infoPort=40361, infoSecurePort=0, ipcPort=34575, storageInfo=lv=-57;cid=testClusterID;nsid=855321071;c=1731960714281), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-18T20:11:56,537 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/098aacef-40a4-13ba-ed62-ee93d3ee3293 2024-11-18T20:11:56,626 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/098aacef-40a4-13ba-ed62-ee93d3ee3293/cluster_08cd2a1d-a16d-5ba4-ed14-6adb17e4da61/zookeeper_0, clientPort=55235, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/098aacef-40a4-13ba-ed62-ee93d3ee3293/cluster_08cd2a1d-a16d-5ba4-ed14-6adb17e4da61/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/098aacef-40a4-13ba-ed62-ee93d3ee3293/cluster_08cd2a1d-a16d-5ba4-ed14-6adb17e4da61/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-18T20:11:56,640 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=55235 2024-11-18T20:11:56,660 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-18T20:11:56,665 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-18T20:11:56,988 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073741825_1001 (size=7) 2024-11-18T20:11:57,403 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123 with version=8 2024-11-18T20:11:57,403 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/hbase-staging 2024-11-18T20:11:57,488 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-18T20:11:57,690 INFO [Time-limited test {}] client.ConnectionUtils(128): master/c0a89b2656d4:0 server-side Connection retries=45 2024-11-18T20:11:57,699 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-18T20:11:57,699 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-18T20:11:57,705 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-18T20:11:57,705 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-18T20:11:57,705 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-18T20:11:57,863 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-18T20:11:57,926 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-18T20:11:57,937 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-18T20:11:57,941 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-18T20:11:57,967 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 1008 (auto-detected) 2024-11-18T20:11:57,968 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-11-18T20:11:57,987 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:41277 2024-11-18T20:11:58,015 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:41277 connecting to ZooKeeper ensemble=127.0.0.1:55235 2024-11-18T20:11:58,050 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:412770x0, quorum=127.0.0.1:55235, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-18T20:11:58,055 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:41277-0x100547699a60000 connected 2024-11-18T20:11:58,093 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-18T20:11:58,097 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-18T20:11:58,109 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:41277-0x100547699a60000, quorum=127.0.0.1:55235, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-18T20:11:58,113 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123, hbase.cluster.distributed=false 2024-11-18T20:11:58,134 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:41277-0x100547699a60000, quorum=127.0.0.1:55235, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-18T20:11:58,139 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=41277 2024-11-18T20:11:58,139 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=41277 2024-11-18T20:11:58,139 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=41277 2024-11-18T20:11:58,145 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=41277 2024-11-18T20:11:58,145 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=41277 2024-11-18T20:11:58,261 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/c0a89b2656d4:0 server-side Connection retries=45 2024-11-18T20:11:58,263 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-18T20:11:58,263 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-18T20:11:58,264 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-18T20:11:58,264 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-18T20:11:58,264 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-18T20:11:58,267 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-18T20:11:58,270 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-18T20:11:58,271 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:35963 2024-11-18T20:11:58,273 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:35963 connecting to ZooKeeper ensemble=127.0.0.1:55235 2024-11-18T20:11:58,274 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-18T20:11:58,278 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-18T20:11:58,286 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:359630x0, quorum=127.0.0.1:55235, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-18T20:11:58,286 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:359630x0, quorum=127.0.0.1:55235, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-18T20:11:58,287 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:35963-0x100547699a60001 connected 2024-11-18T20:11:58,292 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-18T20:11:58,304 DEBUG [Time-limited test {}] mob.MobFileCache(123): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-18T20:11:58,307 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:35963-0x100547699a60001, quorum=127.0.0.1:55235, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-18T20:11:58,312 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:35963-0x100547699a60001, quorum=127.0.0.1:55235, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-18T20:11:58,312 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=35963 2024-11-18T20:11:58,313 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=35963 2024-11-18T20:11:58,313 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=35963 2024-11-18T20:11:58,314 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=35963 2024-11-18T20:11:58,315 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=35963 2024-11-18T20:11:58,332 DEBUG [M:0;c0a89b2656d4:41277 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;c0a89b2656d4:41277 2024-11-18T20:11:58,333 INFO [master/c0a89b2656d4:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/c0a89b2656d4,41277,1731960717534 2024-11-18T20:11:58,339 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35963-0x100547699a60001, quorum=127.0.0.1:55235, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-18T20:11:58,339 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41277-0x100547699a60000, quorum=127.0.0.1:55235, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-18T20:11:58,340 DEBUG [master/c0a89b2656d4:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:41277-0x100547699a60000, quorum=127.0.0.1:55235, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/c0a89b2656d4,41277,1731960717534 2024-11-18T20:11:58,366 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35963-0x100547699a60001, quorum=127.0.0.1:55235, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-18T20:11:58,366 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41277-0x100547699a60000, quorum=127.0.0.1:55235, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T20:11:58,366 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35963-0x100547699a60001, quorum=127.0.0.1:55235, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T20:11:58,367 DEBUG [master/c0a89b2656d4:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:41277-0x100547699a60000, quorum=127.0.0.1:55235, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-18T20:11:58,369 INFO [master/c0a89b2656d4:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/c0a89b2656d4,41277,1731960717534 from backup master directory 2024-11-18T20:11:58,371 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41277-0x100547699a60000, quorum=127.0.0.1:55235, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/c0a89b2656d4,41277,1731960717534 2024-11-18T20:11:58,371 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35963-0x100547699a60001, quorum=127.0.0.1:55235, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-18T20:11:58,372 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41277-0x100547699a60000, quorum=127.0.0.1:55235, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-18T20:11:58,372 WARN [master/c0a89b2656d4:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-18T20:11:58,373 INFO [master/c0a89b2656d4:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=c0a89b2656d4,41277,1731960717534 2024-11-18T20:11:58,375 INFO [master/c0a89b2656d4:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-18T20:11:58,377 INFO [master/c0a89b2656d4:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-18T20:11:58,438 DEBUG [master/c0a89b2656d4:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/hbase.id] with ID: ac99df6b-7791-4693-a81a-6a5a39504490 2024-11-18T20:11:58,438 DEBUG [master/c0a89b2656d4:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/.tmp/hbase.id 2024-11-18T20:11:58,448 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073741826_1002 (size=42) 2024-11-18T20:11:58,851 DEBUG [master/c0a89b2656d4:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/.tmp/hbase.id]:[hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/hbase.id] 2024-11-18T20:11:58,907 INFO [master/c0a89b2656d4:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-18T20:11:58,913 INFO [master/c0a89b2656d4:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-18T20:11:58,934 INFO [master/c0a89b2656d4:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 18ms. 2024-11-18T20:11:58,937 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41277-0x100547699a60000, quorum=127.0.0.1:55235, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T20:11:58,937 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35963-0x100547699a60001, quorum=127.0.0.1:55235, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T20:11:58,952 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073741827_1003 (size=196) 2024-11-18T20:11:59,370 INFO [master/c0a89b2656d4:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-18T20:11:59,374 INFO [master/c0a89b2656d4:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-18T20:11:59,395 DEBUG [master/c0a89b2656d4:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:11:59,400 INFO [master/c0a89b2656d4:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-18T20:11:59,436 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073741828_1004 (size=1189) 2024-11-18T20:11:59,458 INFO [master/c0a89b2656d4:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/MasterData/data/master/store 2024-11-18T20:11:59,482 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073741829_1005 (size=34) 2024-11-18T20:11:59,889 INFO [master/c0a89b2656d4:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-18T20:11:59,893 DEBUG [master/c0a89b2656d4:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T20:11:59,895 DEBUG [master/c0a89b2656d4:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-18T20:11:59,895 INFO [master/c0a89b2656d4:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-18T20:11:59,895 DEBUG [master/c0a89b2656d4:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-18T20:11:59,897 DEBUG [master/c0a89b2656d4:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-18T20:11:59,897 DEBUG [master/c0a89b2656d4:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-18T20:11:59,897 INFO [master/c0a89b2656d4:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-18T20:11:59,898 DEBUG [master/c0a89b2656d4:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731960719895Disabling compacts and flushes for region at 1731960719895Disabling writes for close at 1731960719897 (+2 ms)Writing region close event to WAL at 1731960719897Closed at 1731960719897 2024-11-18T20:11:59,901 WARN [master/c0a89b2656d4:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/MasterData/data/master/store/.initializing 2024-11-18T20:11:59,901 DEBUG [master/c0a89b2656d4:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/MasterData/WALs/c0a89b2656d4,41277,1731960717534 2024-11-18T20:11:59,912 INFO [master/c0a89b2656d4:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-18T20:11:59,926 INFO [master/c0a89b2656d4:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=c0a89b2656d4%2C41277%2C1731960717534, suffix=, logDir=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/MasterData/WALs/c0a89b2656d4,41277,1731960717534, archiveDir=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/MasterData/oldWALs, maxLogs=10 2024-11-18T20:11:59,956 DEBUG [master/c0a89b2656d4:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/MasterData/WALs/c0a89b2656d4,41277,1731960717534/c0a89b2656d4%2C41277%2C1731960717534.1731960719932, exclude list is [], retry=0 2024-11-18T20:11:59,971 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41017,DS-d929842b-b907-444e-b62d-3ef7011a945c,DISK] 2024-11-18T20:11:59,974 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-11-18T20:12:00,005 INFO [master/c0a89b2656d4:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/MasterData/WALs/c0a89b2656d4,41277,1731960717534/c0a89b2656d4%2C41277%2C1731960717534.1731960719932 2024-11-18T20:12:00,006 DEBUG [master/c0a89b2656d4:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40361:40361)] 2024-11-18T20:12:00,007 DEBUG [master/c0a89b2656d4:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-18T20:12:00,007 DEBUG [master/c0a89b2656d4:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T20:12:00,010 DEBUG [master/c0a89b2656d4:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-18T20:12:00,011 DEBUG [master/c0a89b2656d4:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-18T20:12:00,045 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-18T20:12:00,070 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-18T20:12:00,074 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:00,077 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-18T20:12:00,078 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-18T20:12:00,081 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-18T20:12:00,082 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:00,083 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T20:12:00,083 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-18T20:12:00,087 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-18T20:12:00,087 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:00,089 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T20:12:00,089 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-18T20:12:00,092 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-18T20:12:00,092 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:00,094 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T20:12:00,094 DEBUG [master/c0a89b2656d4:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-18T20:12:00,098 DEBUG [master/c0a89b2656d4:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-18T20:12:00,099 DEBUG [master/c0a89b2656d4:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-18T20:12:00,104 DEBUG [master/c0a89b2656d4:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-18T20:12:00,105 DEBUG [master/c0a89b2656d4:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-18T20:12:00,109 DEBUG [master/c0a89b2656d4:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-18T20:12:00,114 DEBUG [master/c0a89b2656d4:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-18T20:12:00,122 DEBUG [master/c0a89b2656d4:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-18T20:12:00,124 INFO [master/c0a89b2656d4:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72783973, jitterRate=0.08456571400165558}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-18T20:12:00,134 DEBUG [master/c0a89b2656d4:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1731960720022Initializing all the Stores at 1731960720025 (+3 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731960720025Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731960720026 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731960720026Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731960720026Cleaning up temporary data from old regions at 1731960720105 (+79 ms)Region opened successfully at 1731960720134 (+29 ms) 2024-11-18T20:12:00,136 INFO [master/c0a89b2656d4:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-18T20:12:00,176 DEBUG [master/c0a89b2656d4:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5766a8ac, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=c0a89b2656d4/172.17.0.2:0 2024-11-18T20:12:00,216 INFO [master/c0a89b2656d4:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-18T20:12:00,227 INFO [master/c0a89b2656d4:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-18T20:12:00,227 INFO [master/c0a89b2656d4:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-18T20:12:00,230 INFO [master/c0a89b2656d4:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-18T20:12:00,232 INFO [master/c0a89b2656d4:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-18T20:12:00,238 INFO [master/c0a89b2656d4:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 5 msec 2024-11-18T20:12:00,238 INFO [master/c0a89b2656d4:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-18T20:12:00,272 INFO [master/c0a89b2656d4:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-18T20:12:00,287 DEBUG [master/c0a89b2656d4:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:41277-0x100547699a60000, quorum=127.0.0.1:55235, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-18T20:12:00,289 DEBUG [master/c0a89b2656d4:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-18T20:12:00,292 INFO [master/c0a89b2656d4:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-18T20:12:00,294 DEBUG [master/c0a89b2656d4:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:41277-0x100547699a60000, quorum=127.0.0.1:55235, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-18T20:12:00,295 DEBUG [master/c0a89b2656d4:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-18T20:12:00,297 INFO [master/c0a89b2656d4:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-18T20:12:00,300 DEBUG [master/c0a89b2656d4:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:41277-0x100547699a60000, quorum=127.0.0.1:55235, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-18T20:12:00,302 DEBUG [master/c0a89b2656d4:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-18T20:12:00,303 DEBUG [master/c0a89b2656d4:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:41277-0x100547699a60000, quorum=127.0.0.1:55235, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-18T20:12:00,304 DEBUG [master/c0a89b2656d4:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-18T20:12:00,322 DEBUG [master/c0a89b2656d4:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:41277-0x100547699a60000, quorum=127.0.0.1:55235, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-18T20:12:00,323 DEBUG [master/c0a89b2656d4:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-18T20:12:00,329 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35963-0x100547699a60001, quorum=127.0.0.1:55235, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-18T20:12:00,329 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41277-0x100547699a60000, quorum=127.0.0.1:55235, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-18T20:12:00,329 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35963-0x100547699a60001, quorum=127.0.0.1:55235, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T20:12:00,329 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41277-0x100547699a60000, quorum=127.0.0.1:55235, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T20:12:00,332 INFO [master/c0a89b2656d4:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=c0a89b2656d4,41277,1731960717534, sessionid=0x100547699a60000, setting cluster-up flag (Was=false) 2024-11-18T20:12:00,344 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41277-0x100547699a60000, quorum=127.0.0.1:55235, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T20:12:00,344 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35963-0x100547699a60001, quorum=127.0.0.1:55235, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T20:12:00,350 DEBUG [master/c0a89b2656d4:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-18T20:12:00,352 DEBUG [master/c0a89b2656d4:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=c0a89b2656d4,41277,1731960717534 2024-11-18T20:12:00,357 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35963-0x100547699a60001, quorum=127.0.0.1:55235, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T20:12:00,357 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41277-0x100547699a60000, quorum=127.0.0.1:55235, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T20:12:00,364 DEBUG [master/c0a89b2656d4:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-18T20:12:00,366 DEBUG [master/c0a89b2656d4:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=c0a89b2656d4,41277,1731960717534 2024-11-18T20:12:00,373 INFO [master/c0a89b2656d4:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-18T20:12:00,423 INFO [RS:0;c0a89b2656d4:35963 {}] regionserver.HRegionServer(746): ClusterId : ac99df6b-7791-4693-a81a-6a5a39504490 2024-11-18T20:12:00,426 DEBUG [RS:0;c0a89b2656d4:35963 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-18T20:12:00,437 DEBUG [RS:0;c0a89b2656d4:35963 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-18T20:12:00,437 DEBUG [RS:0;c0a89b2656d4:35963 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-18T20:12:00,440 DEBUG [RS:0;c0a89b2656d4:35963 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-18T20:12:00,440 DEBUG [RS:0;c0a89b2656d4:35963 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@322deac5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=c0a89b2656d4/172.17.0.2:0 2024-11-18T20:12:00,456 DEBUG [RS:0;c0a89b2656d4:35963 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;c0a89b2656d4:35963 2024-11-18T20:12:00,459 INFO [RS:0;c0a89b2656d4:35963 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-18T20:12:00,459 INFO [RS:0;c0a89b2656d4:35963 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-18T20:12:00,459 DEBUG [RS:0;c0a89b2656d4:35963 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-18T20:12:00,462 INFO [RS:0;c0a89b2656d4:35963 {}] regionserver.HRegionServer(2659): reportForDuty to master=c0a89b2656d4,41277,1731960717534 with port=35963, startcode=1731960718223 2024-11-18T20:12:00,475 DEBUG [master/c0a89b2656d4:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-18T20:12:00,478 DEBUG [RS:0;c0a89b2656d4:35963 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-18T20:12:00,492 INFO [master/c0a89b2656d4:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-18T20:12:00,501 INFO [master/c0a89b2656d4:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-18T20:12:00,509 DEBUG [master/c0a89b2656d4:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: c0a89b2656d4,41277,1731960717534 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-18T20:12:00,519 DEBUG [master/c0a89b2656d4:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/c0a89b2656d4:0, corePoolSize=5, maxPoolSize=5 2024-11-18T20:12:00,519 DEBUG [master/c0a89b2656d4:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/c0a89b2656d4:0, corePoolSize=5, maxPoolSize=5 2024-11-18T20:12:00,520 DEBUG [master/c0a89b2656d4:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/c0a89b2656d4:0, corePoolSize=5, maxPoolSize=5 2024-11-18T20:12:00,520 DEBUG [master/c0a89b2656d4:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/c0a89b2656d4:0, corePoolSize=5, maxPoolSize=5 2024-11-18T20:12:00,520 DEBUG [master/c0a89b2656d4:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/c0a89b2656d4:0, corePoolSize=10, maxPoolSize=10 2024-11-18T20:12:00,520 DEBUG [master/c0a89b2656d4:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/c0a89b2656d4:0, corePoolSize=1, maxPoolSize=1 2024-11-18T20:12:00,520 DEBUG [master/c0a89b2656d4:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/c0a89b2656d4:0, corePoolSize=2, maxPoolSize=2 2024-11-18T20:12:00,521 DEBUG [master/c0a89b2656d4:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/c0a89b2656d4:0, corePoolSize=1, maxPoolSize=1 2024-11-18T20:12:00,529 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-18T20:12:00,533 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-18T20:12:00,544 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:00,545 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-18T20:12:00,549 INFO [master/c0a89b2656d4:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731960750549 2024-11-18T20:12:00,551 INFO [master/c0a89b2656d4:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-18T20:12:00,553 INFO [master/c0a89b2656d4:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-18T20:12:00,561 INFO [master/c0a89b2656d4:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-18T20:12:00,562 INFO [master/c0a89b2656d4:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-18T20:12:00,563 INFO [master/c0a89b2656d4:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-18T20:12:00,563 INFO [master/c0a89b2656d4:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-18T20:12:00,563 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46819, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-11-18T20:12:00,567 INFO [master/c0a89b2656d4:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-18T20:12:00,571 INFO [master/c0a89b2656d4:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-18T20:12:00,572 INFO [master/c0a89b2656d4:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-18T20:12:00,572 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=41277 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:667) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16714) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:00,573 INFO [master/c0a89b2656d4:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-18T20:12:00,577 INFO [master/c0a89b2656d4:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-18T20:12:00,577 INFO [master/c0a89b2656d4:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-18T20:12:00,578 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073741831_1007 (size=1321) 2024-11-18T20:12:00,581 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-18T20:12:00,582 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123 2024-11-18T20:12:00,583 DEBUG [master/c0a89b2656d4:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/c0a89b2656d4:0:becomeActiveMaster-HFileCleaner.large.0-1731960720582,5,FailOnTimeoutGroup] 2024-11-18T20:12:00,585 DEBUG [master/c0a89b2656d4:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/c0a89b2656d4:0:becomeActiveMaster-HFileCleaner.small.0-1731960720583,5,FailOnTimeoutGroup] 2024-11-18T20:12:00,585 INFO [master/c0a89b2656d4:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-18T20:12:00,586 INFO [master/c0a89b2656d4:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-18T20:12:00,587 INFO [master/c0a89b2656d4:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-18T20:12:00,588 INFO [master/c0a89b2656d4:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-18T20:12:00,602 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073741832_1008 (size=32) 2024-11-18T20:12:00,606 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T20:12:00,613 DEBUG [RS:0;c0a89b2656d4:35963 {}] regionserver.HRegionServer(2683): Master is not running yet 2024-11-18T20:12:00,613 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-18T20:12:00,613 WARN [RS:0;c0a89b2656d4:35963 {}] regionserver.HRegionServer(841): reportForDuty failed; sleeping 100 ms and then retrying. 2024-11-18T20:12:00,617 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-18T20:12:00,617 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:00,618 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-18T20:12:00,619 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-18T20:12:00,622 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-18T20:12:00,623 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:00,624 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-18T20:12:00,624 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-18T20:12:00,627 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-18T20:12:00,628 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:00,629 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-18T20:12:00,629 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-18T20:12:00,632 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-18T20:12:00,633 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:00,634 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-18T20:12:00,634 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-18T20:12:00,636 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/hbase/meta/1588230740 2024-11-18T20:12:00,637 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/hbase/meta/1588230740 2024-11-18T20:12:00,640 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-18T20:12:00,640 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-18T20:12:00,641 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-18T20:12:00,644 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-18T20:12:00,648 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-18T20:12:00,649 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63253650, jitterRate=-0.0574471652507782}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-18T20:12:00,655 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1731960720606Initializing all the Stores at 1731960720609 (+3 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731960720609Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731960720613 (+4 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731960720613Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731960720613Cleaning up temporary data from old regions at 1731960720640 (+27 ms)Region opened successfully at 1731960720655 (+15 ms) 2024-11-18T20:12:00,655 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-18T20:12:00,656 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-18T20:12:00,656 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-18T20:12:00,656 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-18T20:12:00,656 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-18T20:12:00,658 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-18T20:12:00,658 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731960720655Disabling compacts and flushes for region at 1731960720655Disabling writes for close at 1731960720656 (+1 ms)Writing region close event to WAL at 1731960720657 (+1 ms)Closed at 1731960720658 (+1 ms) 2024-11-18T20:12:00,662 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-18T20:12:00,662 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-18T20:12:00,668 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-18T20:12:00,675 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-18T20:12:00,680 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-18T20:12:00,715 INFO [RS:0;c0a89b2656d4:35963 {}] regionserver.HRegionServer(2659): reportForDuty to master=c0a89b2656d4,41277,1731960717534 with port=35963, startcode=1731960718223 2024-11-18T20:12:00,718 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=41277 {}] master.ServerManager(363): Checking decommissioned status of RegionServer c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:00,722 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=41277 {}] master.ServerManager(517): Registering regionserver=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:00,731 DEBUG [RS:0;c0a89b2656d4:35963 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123 2024-11-18T20:12:00,731 DEBUG [RS:0;c0a89b2656d4:35963 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:44393 2024-11-18T20:12:00,731 DEBUG [RS:0;c0a89b2656d4:35963 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-18T20:12:00,735 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41277-0x100547699a60000, quorum=127.0.0.1:55235, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-18T20:12:00,736 DEBUG [RS:0;c0a89b2656d4:35963 {}] zookeeper.ZKUtil(111): regionserver:35963-0x100547699a60001, quorum=127.0.0.1:55235, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:00,736 WARN [RS:0;c0a89b2656d4:35963 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-18T20:12:00,736 INFO [RS:0;c0a89b2656d4:35963 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-18T20:12:00,737 DEBUG [RS:0;c0a89b2656d4:35963 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/WALs/c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:00,739 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [c0a89b2656d4,35963,1731960718223] 2024-11-18T20:12:00,765 INFO [RS:0;c0a89b2656d4:35963 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-18T20:12:00,781 INFO [RS:0;c0a89b2656d4:35963 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-18T20:12:00,787 INFO [RS:0;c0a89b2656d4:35963 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-18T20:12:00,787 INFO [RS:0;c0a89b2656d4:35963 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-18T20:12:00,788 INFO [RS:0;c0a89b2656d4:35963 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-18T20:12:00,792 INFO [RS:0;c0a89b2656d4:35963 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-18T20:12:00,794 INFO [RS:0;c0a89b2656d4:35963 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-18T20:12:00,794 DEBUG [RS:0;c0a89b2656d4:35963 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/c0a89b2656d4:0, corePoolSize=1, maxPoolSize=1 2024-11-18T20:12:00,794 DEBUG [RS:0;c0a89b2656d4:35963 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/c0a89b2656d4:0, corePoolSize=1, maxPoolSize=1 2024-11-18T20:12:00,794 DEBUG [RS:0;c0a89b2656d4:35963 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/c0a89b2656d4:0, corePoolSize=1, maxPoolSize=1 2024-11-18T20:12:00,795 DEBUG [RS:0;c0a89b2656d4:35963 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/c0a89b2656d4:0, corePoolSize=1, maxPoolSize=1 2024-11-18T20:12:00,795 DEBUG [RS:0;c0a89b2656d4:35963 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/c0a89b2656d4:0, corePoolSize=1, maxPoolSize=1 2024-11-18T20:12:00,795 DEBUG [RS:0;c0a89b2656d4:35963 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/c0a89b2656d4:0, corePoolSize=2, maxPoolSize=2 2024-11-18T20:12:00,795 DEBUG [RS:0;c0a89b2656d4:35963 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/c0a89b2656d4:0, corePoolSize=1, maxPoolSize=1 2024-11-18T20:12:00,795 DEBUG [RS:0;c0a89b2656d4:35963 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/c0a89b2656d4:0, corePoolSize=1, maxPoolSize=1 2024-11-18T20:12:00,795 DEBUG [RS:0;c0a89b2656d4:35963 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/c0a89b2656d4:0, corePoolSize=1, maxPoolSize=1 2024-11-18T20:12:00,796 DEBUG [RS:0;c0a89b2656d4:35963 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/c0a89b2656d4:0, corePoolSize=1, maxPoolSize=1 2024-11-18T20:12:00,796 DEBUG [RS:0;c0a89b2656d4:35963 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/c0a89b2656d4:0, corePoolSize=1, maxPoolSize=1 2024-11-18T20:12:00,796 DEBUG [RS:0;c0a89b2656d4:35963 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/c0a89b2656d4:0, corePoolSize=1, maxPoolSize=1 2024-11-18T20:12:00,796 DEBUG [RS:0;c0a89b2656d4:35963 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/c0a89b2656d4:0, corePoolSize=3, maxPoolSize=3 2024-11-18T20:12:00,796 DEBUG [RS:0;c0a89b2656d4:35963 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0, corePoolSize=3, maxPoolSize=3 2024-11-18T20:12:00,797 INFO [RS:0;c0a89b2656d4:35963 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-18T20:12:00,797 INFO [RS:0;c0a89b2656d4:35963 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-18T20:12:00,798 INFO [RS:0;c0a89b2656d4:35963 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-18T20:12:00,798 INFO [RS:0;c0a89b2656d4:35963 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-18T20:12:00,798 INFO [RS:0;c0a89b2656d4:35963 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-18T20:12:00,798 INFO [RS:0;c0a89b2656d4:35963 {}] hbase.ChoreService(168): Chore ScheduledChore name=c0a89b2656d4,35963,1731960718223-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-18T20:12:00,817 INFO [RS:0;c0a89b2656d4:35963 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-18T20:12:00,819 INFO [RS:0;c0a89b2656d4:35963 {}] hbase.ChoreService(168): Chore ScheduledChore name=c0a89b2656d4,35963,1731960718223-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-18T20:12:00,819 INFO [RS:0;c0a89b2656d4:35963 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-18T20:12:00,820 INFO [RS:0;c0a89b2656d4:35963 {}] regionserver.Replication(171): c0a89b2656d4,35963,1731960718223 started 2024-11-18T20:12:00,831 WARN [c0a89b2656d4:41277 {}] assignment.AssignmentManager(2443): No servers available; cannot place 1 unassigned regions. 2024-11-18T20:12:00,838 INFO [RS:0;c0a89b2656d4:35963 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-18T20:12:00,839 INFO [RS:0;c0a89b2656d4:35963 {}] regionserver.HRegionServer(1482): Serving as c0a89b2656d4,35963,1731960718223, RpcServer on c0a89b2656d4/172.17.0.2:35963, sessionid=0x100547699a60001 2024-11-18T20:12:00,839 DEBUG [RS:0;c0a89b2656d4:35963 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-18T20:12:00,840 DEBUG [RS:0;c0a89b2656d4:35963 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:00,840 DEBUG [RS:0;c0a89b2656d4:35963 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'c0a89b2656d4,35963,1731960718223' 2024-11-18T20:12:00,840 DEBUG [RS:0;c0a89b2656d4:35963 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-18T20:12:00,841 DEBUG [RS:0;c0a89b2656d4:35963 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-18T20:12:00,842 DEBUG [RS:0;c0a89b2656d4:35963 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-18T20:12:00,842 DEBUG [RS:0;c0a89b2656d4:35963 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-18T20:12:00,842 DEBUG [RS:0;c0a89b2656d4:35963 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:00,842 DEBUG [RS:0;c0a89b2656d4:35963 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'c0a89b2656d4,35963,1731960718223' 2024-11-18T20:12:00,842 DEBUG [RS:0;c0a89b2656d4:35963 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-18T20:12:00,843 DEBUG [RS:0;c0a89b2656d4:35963 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-18T20:12:00,843 DEBUG [RS:0;c0a89b2656d4:35963 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-18T20:12:00,844 INFO [RS:0;c0a89b2656d4:35963 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-18T20:12:00,844 INFO [RS:0;c0a89b2656d4:35963 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-18T20:12:00,949 INFO [RS:0;c0a89b2656d4:35963 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-18T20:12:00,952 INFO [RS:0;c0a89b2656d4:35963 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=c0a89b2656d4%2C35963%2C1731960718223, suffix=, logDir=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/WALs/c0a89b2656d4,35963,1731960718223, archiveDir=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/oldWALs, maxLogs=32 2024-11-18T20:12:00,968 DEBUG [RS:0;c0a89b2656d4:35963 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/WALs/c0a89b2656d4,35963,1731960718223/c0a89b2656d4%2C35963%2C1731960718223.1731960720955, exclude list is [], retry=0 2024-11-18T20:12:00,973 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41017,DS-d929842b-b907-444e-b62d-3ef7011a945c,DISK] 2024-11-18T20:12:00,976 INFO [RS:0;c0a89b2656d4:35963 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/WALs/c0a89b2656d4,35963,1731960718223/c0a89b2656d4%2C35963%2C1731960718223.1731960720955 2024-11-18T20:12:00,976 DEBUG [RS:0;c0a89b2656d4:35963 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40361:40361)] 2024-11-18T20:12:01,084 DEBUG [c0a89b2656d4:41277 {}] assignment.AssignmentManager(2464): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-11-18T20:12:01,099 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:01,108 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as c0a89b2656d4,35963,1731960718223, state=OPENING 2024-11-18T20:12:01,114 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-18T20:12:01,116 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41277-0x100547699a60000, quorum=127.0.0.1:55235, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T20:12:01,116 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35963-0x100547699a60001, quorum=127.0.0.1:55235, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T20:12:01,117 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-18T20:12:01,118 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-18T20:12:01,120 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-18T20:12:01,122 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=c0a89b2656d4,35963,1731960718223}] 2024-11-18T20:12:01,306 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-18T20:12:01,310 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43149, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-18T20:12:01,320 INFO [RS_OPEN_META-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-18T20:12:01,320 INFO [RS_OPEN_META-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-18T20:12:01,321 INFO [RS_OPEN_META-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-18T20:12:01,324 INFO [RS_OPEN_META-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=c0a89b2656d4%2C35963%2C1731960718223.meta, suffix=.meta, logDir=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/WALs/c0a89b2656d4,35963,1731960718223, archiveDir=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/oldWALs, maxLogs=32 2024-11-18T20:12:01,340 DEBUG [RS_OPEN_META-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/WALs/c0a89b2656d4,35963,1731960718223/c0a89b2656d4%2C35963%2C1731960718223.meta.1731960721326.meta, exclude list is [], retry=0 2024-11-18T20:12:01,344 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41017,DS-d929842b-b907-444e-b62d-3ef7011a945c,DISK] 2024-11-18T20:12:01,348 INFO [RS_OPEN_META-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/WALs/c0a89b2656d4,35963,1731960718223/c0a89b2656d4%2C35963%2C1731960718223.meta.1731960721326.meta 2024-11-18T20:12:01,348 DEBUG [RS_OPEN_META-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40361:40361)] 2024-11-18T20:12:01,348 DEBUG [RS_OPEN_META-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-18T20:12:01,350 DEBUG [RS_OPEN_META-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-18T20:12:01,353 DEBUG [RS_OPEN_META-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-18T20:12:01,357 INFO [RS_OPEN_META-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-18T20:12:01,361 DEBUG [RS_OPEN_META-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-18T20:12:01,362 DEBUG [RS_OPEN_META-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T20:12:01,362 DEBUG [RS_OPEN_META-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-18T20:12:01,362 DEBUG [RS_OPEN_META-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-18T20:12:01,366 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-18T20:12:01,368 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-18T20:12:01,368 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:01,369 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-18T20:12:01,369 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-18T20:12:01,371 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-18T20:12:01,371 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:01,372 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-18T20:12:01,372 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-18T20:12:01,374 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-18T20:12:01,374 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:01,375 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-18T20:12:01,375 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-18T20:12:01,376 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-18T20:12:01,377 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:01,377 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-18T20:12:01,378 DEBUG [RS_OPEN_META-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-18T20:12:01,380 DEBUG [RS_OPEN_META-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/hbase/meta/1588230740 2024-11-18T20:12:01,383 DEBUG [RS_OPEN_META-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/hbase/meta/1588230740 2024-11-18T20:12:01,385 DEBUG [RS_OPEN_META-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-18T20:12:01,386 DEBUG [RS_OPEN_META-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-18T20:12:01,387 DEBUG [RS_OPEN_META-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-18T20:12:01,389 DEBUG [RS_OPEN_META-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-18T20:12:01,391 INFO [RS_OPEN_META-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60221161, jitterRate=-0.10263477265834808}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-18T20:12:01,391 DEBUG [RS_OPEN_META-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-18T20:12:01,394 DEBUG [RS_OPEN_META-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1731960721363Writing region info on filesystem at 1731960721363Initializing all the Stores at 1731960721365 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731960721365Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731960721366 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731960721366Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731960721366Cleaning up temporary data from old regions at 1731960721386 (+20 ms)Running coprocessor post-open hooks at 1731960721392 (+6 ms)Region opened successfully at 1731960721393 (+1 ms) 2024-11-18T20:12:01,401 INFO [RS_OPEN_META-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1731960721296 2024-11-18T20:12:01,411 DEBUG [RS_OPEN_META-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-18T20:12:01,412 INFO [RS_OPEN_META-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-18T20:12:01,414 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:01,417 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as c0a89b2656d4,35963,1731960718223, state=OPEN 2024-11-18T20:12:01,420 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41277-0x100547699a60000, quorum=127.0.0.1:55235, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-18T20:12:01,420 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35963-0x100547699a60001, quorum=127.0.0.1:55235, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-18T20:12:01,421 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-18T20:12:01,421 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-18T20:12:01,421 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:01,427 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-18T20:12:01,427 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=c0a89b2656d4,35963,1731960718223 in 299 msec 2024-11-18T20:12:01,435 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-18T20:12:01,435 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 760 msec 2024-11-18T20:12:01,437 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-18T20:12:01,437 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-18T20:12:01,459 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T20:12:01,461 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c0a89b2656d4,35963,1731960718223, seqNum=-1] 2024-11-18T20:12:01,489 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T20:12:01,493 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36043, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T20:12:01,528 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.1090 sec 2024-11-18T20:12:01,528 INFO [master/c0a89b2656d4:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1731960721528, completionTime=-1 2024-11-18T20:12:01,532 INFO [master/c0a89b2656d4:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-11-18T20:12:01,532 DEBUG [master/c0a89b2656d4:0:becomeActiveMaster {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-18T20:12:01,560 INFO [master/c0a89b2656d4:0:becomeActiveMaster {}] assignment.AssignmentManager(1768): Number of RegionServers=1 2024-11-18T20:12:01,560 INFO [master/c0a89b2656d4:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731960781560 2024-11-18T20:12:01,560 INFO [master/c0a89b2656d4:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731960841560 2024-11-18T20:12:01,560 INFO [master/c0a89b2656d4:0:becomeActiveMaster {}] assignment.AssignmentManager(1775): Joined the cluster in 28 msec 2024-11-18T20:12:01,562 INFO [master/c0a89b2656d4:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=c0a89b2656d4,41277,1731960717534-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-18T20:12:01,563 INFO [master/c0a89b2656d4:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=c0a89b2656d4,41277,1731960717534-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-18T20:12:01,563 INFO [master/c0a89b2656d4:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=c0a89b2656d4,41277,1731960717534-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-18T20:12:01,564 INFO [master/c0a89b2656d4:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-c0a89b2656d4:41277, period=300000, unit=MILLISECONDS is enabled. 2024-11-18T20:12:01,565 INFO [master/c0a89b2656d4:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-18T20:12:01,565 INFO [master/c0a89b2656d4:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-18T20:12:01,571 DEBUG [master/c0a89b2656d4:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-18T20:12:01,593 INFO [master/c0a89b2656d4:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 3.220sec 2024-11-18T20:12:01,595 INFO [master/c0a89b2656d4:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-18T20:12:01,596 INFO [master/c0a89b2656d4:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-18T20:12:01,597 INFO [master/c0a89b2656d4:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-18T20:12:01,598 INFO [master/c0a89b2656d4:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-18T20:12:01,598 INFO [master/c0a89b2656d4:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-18T20:12:01,599 INFO [master/c0a89b2656d4:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=c0a89b2656d4,41277,1731960717534-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-18T20:12:01,599 INFO [master/c0a89b2656d4:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=c0a89b2656d4,41277,1731960717534-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-18T20:12:01,608 DEBUG [master/c0a89b2656d4:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-18T20:12:01,609 INFO [master/c0a89b2656d4:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-18T20:12:01,609 INFO [master/c0a89b2656d4:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=c0a89b2656d4,41277,1731960717534-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-18T20:12:01,632 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@25dfad15, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:12:01,634 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-18T20:12:01,634 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-18T20:12:01,636 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c0a89b2656d4,41277,-1 for getting cluster id 2024-11-18T20:12:01,638 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T20:12:01,645 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ac99df6b-7791-4693-a81a-6a5a39504490' 2024-11-18T20:12:01,647 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T20:12:01,647 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ac99df6b-7791-4693-a81a-6a5a39504490" 2024-11-18T20:12:01,649 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@40f7a663, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:12:01,649 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c0a89b2656d4,41277,-1] 2024-11-18T20:12:01,651 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T20:12:01,653 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:12:01,655 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59712, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T20:12:01,658 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@46b5c665, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:12:01,659 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T20:12:01,666 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c0a89b2656d4,35963,1731960718223, seqNum=-1] 2024-11-18T20:12:01,667 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T20:12:01,670 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33014, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T20:12:01,694 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=c0a89b2656d4,41277,1731960717534 2024-11-18T20:12:01,708 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testMixedAtomicity Thread=210, OpenFileDescriptor=441, MaxFileDescriptor=1048576, SystemLoadAverage=312, ProcessCount=11, AvailableMemoryMB=3789 2024-11-18T20:12:01,718 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-18T20:12:01,723 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncConnectionImpl(321): The fetched master address is c0a89b2656d4,41277,1731960717534 2024-11-18T20:12:01,725 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@4f9587a2 2024-11-18T20:12:01,727 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-18T20:12:01,730 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59726, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-18T20:12:01,732 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41277 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-18T20:12:01,736 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41277 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-18T20:12:01,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41277 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-18T20:12:01,747 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-18T20:12:01,749 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41277 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 4 2024-11-18T20:12:01,750 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:01,752 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-18T20:12:01,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-18T20:12:01,763 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073741835_1011 (size=959) 2024-11-18T20:12:01,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-18T20:12:02,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-18T20:12:02,167 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 326884d1d40acf488e8bd50aea838c73, NAME => 'TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123 2024-11-18T20:12:02,177 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073741836_1012 (size=53) 2024-11-18T20:12:02,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-18T20:12:02,578 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T20:12:02,579 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 326884d1d40acf488e8bd50aea838c73, disabling compactions & flushes 2024-11-18T20:12:02,579 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73. 2024-11-18T20:12:02,579 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73. 2024-11-18T20:12:02,579 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73. after waiting 0 ms 2024-11-18T20:12:02,579 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73. 2024-11-18T20:12:02,579 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73. 2024-11-18T20:12:02,579 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 326884d1d40acf488e8bd50aea838c73: Waiting for close lock at 1731960722579Disabling compacts and flushes for region at 1731960722579Disabling writes for close at 1731960722579Writing region close event to WAL at 1731960722579Closed at 1731960722579 2024-11-18T20:12:02,582 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-18T20:12:02,588 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731960722582"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731960722582"}]},"ts":"1731960722582"} 2024-11-18T20:12:02,593 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-18T20:12:02,595 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-18T20:12:02,597 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731960722595"}]},"ts":"1731960722595"} 2024-11-18T20:12:02,602 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-18T20:12:02,604 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=326884d1d40acf488e8bd50aea838c73, ASSIGN}] 2024-11-18T20:12:02,606 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=326884d1d40acf488e8bd50aea838c73, ASSIGN 2024-11-18T20:12:02,608 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=326884d1d40acf488e8bd50aea838c73, ASSIGN; state=OFFLINE, location=c0a89b2656d4,35963,1731960718223; forceNewPlan=false, retain=false 2024-11-18T20:12:02,760 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=326884d1d40acf488e8bd50aea838c73, regionState=OPENING, regionLocation=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:02,766 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=326884d1d40acf488e8bd50aea838c73, ASSIGN because future has completed 2024-11-18T20:12:02,767 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223}] 2024-11-18T20:12:02,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-18T20:12:02,929 INFO [RS_OPEN_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73. 2024-11-18T20:12:02,930 DEBUG [RS_OPEN_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 326884d1d40acf488e8bd50aea838c73, NAME => 'TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73.', STARTKEY => '', ENDKEY => ''} 2024-11-18T20:12:02,930 DEBUG [RS_OPEN_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 326884d1d40acf488e8bd50aea838c73 2024-11-18T20:12:02,930 DEBUG [RS_OPEN_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T20:12:02,931 DEBUG [RS_OPEN_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 326884d1d40acf488e8bd50aea838c73 2024-11-18T20:12:02,931 DEBUG [RS_OPEN_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 326884d1d40acf488e8bd50aea838c73 2024-11-18T20:12:02,934 INFO [StoreOpener-326884d1d40acf488e8bd50aea838c73-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 326884d1d40acf488e8bd50aea838c73 2024-11-18T20:12:02,937 INFO [StoreOpener-326884d1d40acf488e8bd50aea838c73-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 326884d1d40acf488e8bd50aea838c73 columnFamilyName A 2024-11-18T20:12:02,937 DEBUG [StoreOpener-326884d1d40acf488e8bd50aea838c73-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:02,939 INFO [StoreOpener-326884d1d40acf488e8bd50aea838c73-1 {}] regionserver.HStore(327): Store=326884d1d40acf488e8bd50aea838c73/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T20:12:02,939 INFO [StoreOpener-326884d1d40acf488e8bd50aea838c73-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 326884d1d40acf488e8bd50aea838c73 2024-11-18T20:12:02,943 INFO [StoreOpener-326884d1d40acf488e8bd50aea838c73-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 326884d1d40acf488e8bd50aea838c73 columnFamilyName B 2024-11-18T20:12:02,943 DEBUG [StoreOpener-326884d1d40acf488e8bd50aea838c73-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:02,945 INFO [StoreOpener-326884d1d40acf488e8bd50aea838c73-1 {}] regionserver.HStore(327): Store=326884d1d40acf488e8bd50aea838c73/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T20:12:02,945 INFO [StoreOpener-326884d1d40acf488e8bd50aea838c73-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 326884d1d40acf488e8bd50aea838c73 2024-11-18T20:12:02,948 INFO [StoreOpener-326884d1d40acf488e8bd50aea838c73-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 326884d1d40acf488e8bd50aea838c73 columnFamilyName C 2024-11-18T20:12:02,948 DEBUG [StoreOpener-326884d1d40acf488e8bd50aea838c73-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:02,949 INFO [StoreOpener-326884d1d40acf488e8bd50aea838c73-1 {}] regionserver.HStore(327): Store=326884d1d40acf488e8bd50aea838c73/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T20:12:02,950 DEBUG [RS_OPEN_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 326884d1d40acf488e8bd50aea838c73 2024-11-18T20:12:02,952 DEBUG [RS_OPEN_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73 2024-11-18T20:12:02,953 DEBUG [RS_OPEN_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73 2024-11-18T20:12:02,955 DEBUG [RS_OPEN_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 326884d1d40acf488e8bd50aea838c73 2024-11-18T20:12:02,955 DEBUG [RS_OPEN_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 326884d1d40acf488e8bd50aea838c73 2024-11-18T20:12:02,956 DEBUG [RS_OPEN_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-18T20:12:02,958 DEBUG [RS_OPEN_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 326884d1d40acf488e8bd50aea838c73 2024-11-18T20:12:02,962 DEBUG [RS_OPEN_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-18T20:12:02,963 INFO [RS_OPEN_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 326884d1d40acf488e8bd50aea838c73; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70128849, jitterRate=0.045001283288002014}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-18T20:12:02,963 DEBUG [RS_OPEN_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 326884d1d40acf488e8bd50aea838c73 2024-11-18T20:12:02,964 DEBUG [RS_OPEN_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 326884d1d40acf488e8bd50aea838c73: Running coprocessor pre-open hook at 1731960722931Writing region info on filesystem at 1731960722931Initializing all the Stores at 1731960722933 (+2 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731960722933Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731960722934 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731960722934Cleaning up temporary data from old regions at 1731960722955 (+21 ms)Running coprocessor post-open hooks at 1731960722963 (+8 ms)Region opened successfully at 1731960722964 (+1 ms) 2024-11-18T20:12:02,967 INFO [RS_OPEN_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., pid=6, masterSystemTime=1731960722922 2024-11-18T20:12:02,971 DEBUG [RS_OPEN_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73. 2024-11-18T20:12:02,971 INFO [RS_OPEN_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73. 2024-11-18T20:12:02,972 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=326884d1d40acf488e8bd50aea838c73, regionState=OPEN, openSeqNum=2, regionLocation=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:02,974 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=41277 {}] assignment.AssignmentManager(1535): Unable to acquire lock for regionNode state=OPEN, location=c0a89b2656d4,35963,1731960718223, table=TestAcidGuarantees, region=326884d1d40acf488e8bd50aea838c73. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-18T20:12:02,976 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 because future has completed 2024-11-18T20:12:02,983 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-18T20:12:02,985 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 in 211 msec 2024-11-18T20:12:02,988 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-18T20:12:02,988 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=326884d1d40acf488e8bd50aea838c73, ASSIGN in 379 msec 2024-11-18T20:12:02,990 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-18T20:12:02,990 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731960722990"}]},"ts":"1731960722990"} 2024-11-18T20:12:02,994 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-18T20:12:02,997 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-18T20:12:03,000 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.2580 sec 2024-11-18T20:12:03,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-18T20:12:03,905 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-18T20:12:03,910 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@705975cb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:12:03,911 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c0a89b2656d4,41277,-1 for getting cluster id 2024-11-18T20:12:03,911 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T20:12:03,912 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ac99df6b-7791-4693-a81a-6a5a39504490' 2024-11-18T20:12:03,913 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T20:12:03,913 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ac99df6b-7791-4693-a81a-6a5a39504490" 2024-11-18T20:12:03,913 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6e001514, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:12:03,913 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c0a89b2656d4,41277,-1] 2024-11-18T20:12:03,914 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T20:12:03,914 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:12:03,916 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59748, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T20:12:03,917 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@538d0490, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:12:03,918 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T20:12:03,920 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c0a89b2656d4,35963,1731960718223, seqNum=-1] 2024-11-18T20:12:03,920 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T20:12:03,922 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33016, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T20:12:03,924 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-18T20:12:03,926 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is c0a89b2656d4,41277,1731960717534 2024-11-18T20:12:03,926 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@38391cd4 2024-11-18T20:12:03,928 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-18T20:12:03,930 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59754, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-18T20:12:03,934 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6023f6f1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:12:03,934 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c0a89b2656d4,41277,-1 for getting cluster id 2024-11-18T20:12:03,934 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T20:12:03,936 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ac99df6b-7791-4693-a81a-6a5a39504490' 2024-11-18T20:12:03,936 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T20:12:03,937 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ac99df6b-7791-4693-a81a-6a5a39504490" 2024-11-18T20:12:03,937 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4e11900a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:12:03,937 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c0a89b2656d4,41277,-1] 2024-11-18T20:12:03,938 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T20:12:03,938 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:12:03,940 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59776, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T20:12:03,941 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6ca0b5f9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:12:03,942 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@340b46e5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:12:03,942 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c0a89b2656d4,41277,-1 for getting cluster id 2024-11-18T20:12:03,942 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T20:12:03,944 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ac99df6b-7791-4693-a81a-6a5a39504490' 2024-11-18T20:12:03,944 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T20:12:03,944 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ac99df6b-7791-4693-a81a-6a5a39504490" 2024-11-18T20:12:03,945 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@387dfdb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:12:03,945 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c0a89b2656d4,41277,-1] 2024-11-18T20:12:03,945 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T20:12:03,946 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:12:03,947 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59784, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T20:12:03,948 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@25175df8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:12:03,949 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6edf4fef, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:12:03,949 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c0a89b2656d4,41277,-1 for getting cluster id 2024-11-18T20:12:03,949 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T20:12:03,951 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ac99df6b-7791-4693-a81a-6a5a39504490' 2024-11-18T20:12:03,951 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T20:12:03,951 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ac99df6b-7791-4693-a81a-6a5a39504490" 2024-11-18T20:12:03,952 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2f4e6a07, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:12:03,952 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c0a89b2656d4,41277,-1] 2024-11-18T20:12:03,952 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T20:12:03,952 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:12:03,954 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59798, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T20:12:03,955 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5b17c100, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:12:03,956 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@21f4b80, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:12:03,956 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c0a89b2656d4,41277,-1 for getting cluster id 2024-11-18T20:12:03,956 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T20:12:03,957 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ac99df6b-7791-4693-a81a-6a5a39504490' 2024-11-18T20:12:03,958 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T20:12:03,958 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ac99df6b-7791-4693-a81a-6a5a39504490" 2024-11-18T20:12:03,958 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4f338c72, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:12:03,958 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c0a89b2656d4,41277,-1] 2024-11-18T20:12:03,959 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T20:12:03,959 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:12:03,961 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59814, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T20:12:03,962 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@10112b3c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:12:03,963 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@52294ca5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:12:03,963 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c0a89b2656d4,41277,-1 for getting cluster id 2024-11-18T20:12:03,963 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T20:12:03,964 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ac99df6b-7791-4693-a81a-6a5a39504490' 2024-11-18T20:12:03,964 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T20:12:03,964 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ac99df6b-7791-4693-a81a-6a5a39504490" 2024-11-18T20:12:03,965 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@39d3010f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:12:03,965 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c0a89b2656d4,41277,-1] 2024-11-18T20:12:03,965 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T20:12:03,966 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:12:03,967 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59832, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T20:12:03,968 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5adf4095, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:12:03,969 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@388c382, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:12:03,969 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c0a89b2656d4,41277,-1 for getting cluster id 2024-11-18T20:12:03,970 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T20:12:03,971 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ac99df6b-7791-4693-a81a-6a5a39504490' 2024-11-18T20:12:03,971 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T20:12:03,971 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ac99df6b-7791-4693-a81a-6a5a39504490" 2024-11-18T20:12:03,973 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3e154f6d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:12:03,973 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c0a89b2656d4,41277,-1] 2024-11-18T20:12:03,973 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T20:12:03,973 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:12:03,975 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59848, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T20:12:03,976 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3c8578f1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:12:03,977 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1b48425, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:12:03,977 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c0a89b2656d4,41277,-1 for getting cluster id 2024-11-18T20:12:03,977 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T20:12:03,978 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ac99df6b-7791-4693-a81a-6a5a39504490' 2024-11-18T20:12:03,978 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T20:12:03,978 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ac99df6b-7791-4693-a81a-6a5a39504490" 2024-11-18T20:12:03,979 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3bf48cf7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:12:03,979 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c0a89b2656d4,41277,-1] 2024-11-18T20:12:03,979 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T20:12:03,979 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:12:03,981 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59852, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T20:12:03,982 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5d6acfb2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:12:03,983 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5238acdf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:12:03,983 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c0a89b2656d4,41277,-1 for getting cluster id 2024-11-18T20:12:03,983 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T20:12:03,984 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ac99df6b-7791-4693-a81a-6a5a39504490' 2024-11-18T20:12:03,985 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T20:12:03,985 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ac99df6b-7791-4693-a81a-6a5a39504490" 2024-11-18T20:12:03,985 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@12f9cc25, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:12:03,985 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c0a89b2656d4,41277,-1] 2024-11-18T20:12:03,985 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T20:12:03,986 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:12:03,987 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59856, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T20:12:03,988 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@69746154, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:12:03,989 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@74a9c1c7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:12:03,989 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c0a89b2656d4,41277,-1 for getting cluster id 2024-11-18T20:12:03,989 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T20:12:03,990 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ac99df6b-7791-4693-a81a-6a5a39504490' 2024-11-18T20:12:03,991 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T20:12:03,991 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ac99df6b-7791-4693-a81a-6a5a39504490" 2024-11-18T20:12:03,991 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7299e0d7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:12:03,991 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c0a89b2656d4,41277,-1] 2024-11-18T20:12:03,992 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T20:12:03,992 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:12:03,993 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59874, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T20:12:03,994 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@39568e1a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:12:03,998 DEBUG [Thread-156 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T20:12:03,998 DEBUG [Thread-145 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T20:12:03,998 DEBUG [Thread-143 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T20:12:03,999 DEBUG [Thread-147 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T20:12:03,999 DEBUG [Thread-151 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T20:12:03,999 DEBUG [Thread-149 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T20:12:03,999 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c0a89b2656d4,35963,1731960718223, seqNum=-1] 2024-11-18T20:12:04,000 DEBUG [Thread-158 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T20:12:03,999 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c0a89b2656d4,35963,1731960718223, seqNum=-1] 2024-11-18T20:12:04,000 DEBUG [Thread-160 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T20:12:04,000 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T20:12:04,000 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T20:12:04,002 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c0a89b2656d4,35963,1731960718223, seqNum=-1] 2024-11-18T20:12:04,002 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T20:12:04,002 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c0a89b2656d4,35963,1731960718223, seqNum=-1] 2024-11-18T20:12:04,002 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33032, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T20:12:04,003 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T20:12:04,003 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c0a89b2656d4,35963,1731960718223, seqNum=-1] 2024-11-18T20:12:04,003 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c0a89b2656d4,35963,1731960718223, seqNum=-1] 2024-11-18T20:12:04,003 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c0a89b2656d4,35963,1731960718223, seqNum=-1] 2024-11-18T20:12:04,003 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T20:12:04,004 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T20:12:04,004 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T20:12:04,004 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c0a89b2656d4,35963,1731960718223, seqNum=-1] 2024-11-18T20:12:04,005 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T20:12:04,005 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33046, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T20:12:04,006 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33072, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T20:12:04,006 DEBUG [Thread-154 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T20:12:04,006 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33070, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T20:12:04,007 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33088, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T20:12:04,008 DEBUG [Thread-154 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c0a89b2656d4,35963,1731960718223, seqNum=-1] 2024-11-18T20:12:04,009 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33028, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T20:12:04,009 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33056, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T20:12:04,010 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33078, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T20:12:04,010 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41277 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-18T20:12:04,011 DEBUG [Thread-154 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T20:12:04,013 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2] 2024-11-18T20:12:04,013 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33098, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T20:12:04,015 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2] 2024-11-18T20:12:04,015 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2] 2024-11-18T20:12:04,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41277 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees 2024-11-18T20:12:04,017 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2] 2024-11-18T20:12:04,018 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2] 2024-11-18T20:12:04,019 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-18T20:12:04,019 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2] 2024-11-18T20:12:04,022 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2] 2024-11-18T20:12:04,022 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2] 2024-11-18T20:12:04,025 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-18T20:12:04,027 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2] 2024-11-18T20:12:04,027 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-18T20:12:04,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-18T20:12:04,097 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-18T20:12:04,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(8855): Flush requested on 326884d1d40acf488e8bd50aea838c73 2024-11-18T20:12:04,102 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 326884d1d40acf488e8bd50aea838c73 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-18T20:12:04,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-18T20:12:04,227 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35963 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-18T20:12:04,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73. 2024-11-18T20:12:04,240 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73. as already flushing 2024-11-18T20:12:04,247 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/.tmp/A/124683c738434fc8a0d3b04126c95f32 is 50, key is test_row_0/A:col10/1731960724081/Put/seqid=0 2024-11-18T20:12:04,251 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73. 2024-11-18T20:12:04,253 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 326884d1d40acf488e8bd50aea838c73, NAME => 'TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:12:04,267 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 326884d1d40acf488e8bd50aea838c73, NAME => 'TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:12:04,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41277 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 326884d1d40acf488e8bd50aea838c73, NAME => 'TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 326884d1d40acf488e8bd50aea838c73, NAME => 'TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:12:04,300 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073741837_1013 (size=9657) 2024-11-18T20:12:04,302 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/.tmp/A/124683c738434fc8a0d3b04126c95f32 2024-11-18T20:12:04,313 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:04,315 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:04,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33046 deadline: 1731960784287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:04,321 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:04,321 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:04,321 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:12:04,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33056 deadline: 1731960784290, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:04,326 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:04,326 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:04,327 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:12:04,332 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:04,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33070 deadline: 1731960784310, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:04,333 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:04,334 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:04,334 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:12:04,333 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:04,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33072 deadline: 1731960784313, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:04,335 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:04,335 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:04,335 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:12:04,335 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:04,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33028 deadline: 1731960784316, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:04,336 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:04,337 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:04,337 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:12:04,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-18T20:12:04,433 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/.tmp/B/bbffcc63619d435293ea9b31e14d1e5b is 50, key is test_row_0/B:col10/1731960724081/Put/seqid=0 2024-11-18T20:12:04,437 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:04,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33056 deadline: 1731960784435, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:04,439 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:04,439 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:04,439 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:12:04,439 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:04,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33046 deadline: 1731960784436, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:04,441 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:04,441 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:04,441 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:12:04,445 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:04,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33072 deadline: 1731960784444, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:04,447 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:04,447 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:04,447 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:12:04,447 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:04,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33028 deadline: 1731960784445, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:04,447 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:04,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33070 deadline: 1731960784444, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:04,449 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:04,449 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:04,449 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:12:04,449 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:04,449 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:04,449 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:12:04,450 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35963 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-18T20:12:04,459 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073741838_1014 (size=9657) 2024-11-18T20:12:04,460 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/.tmp/B/bbffcc63619d435293ea9b31e14d1e5b 2024-11-18T20:12:04,467 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73. 2024-11-18T20:12:04,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73. as already flushing 2024-11-18T20:12:04,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73. 2024-11-18T20:12:04,468 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 326884d1d40acf488e8bd50aea838c73, NAME => 'TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:12:04,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 326884d1d40acf488e8bd50aea838c73, NAME => 'TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:12:04,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41277 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 326884d1d40acf488e8bd50aea838c73, NAME => 'TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 326884d1d40acf488e8bd50aea838c73, NAME => 'TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:12:04,521 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/.tmp/C/04b4ce1131474dbab3d596c8d7fff1fb is 50, key is test_row_0/C:col10/1731960724081/Put/seqid=0 2024-11-18T20:12:04,532 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073741839_1015 (size=9657) 2024-11-18T20:12:04,538 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/.tmp/C/04b4ce1131474dbab3d596c8d7fff1fb 2024-11-18T20:12:04,561 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/.tmp/A/124683c738434fc8a0d3b04126c95f32 as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/A/124683c738434fc8a0d3b04126c95f32 2024-11-18T20:12:04,587 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/A/124683c738434fc8a0d3b04126c95f32, entries=100, sequenceid=12, filesize=9.4 K 2024-11-18T20:12:04,603 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/.tmp/B/bbffcc63619d435293ea9b31e14d1e5b as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/B/bbffcc63619d435293ea9b31e14d1e5b 2024-11-18T20:12:04,620 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/B/bbffcc63619d435293ea9b31e14d1e5b, entries=100, sequenceid=12, filesize=9.4 K 2024-11-18T20:12:04,623 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/.tmp/C/04b4ce1131474dbab3d596c8d7fff1fb as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/C/04b4ce1131474dbab3d596c8d7fff1fb 2024-11-18T20:12:04,623 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35963 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-18T20:12:04,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73. 2024-11-18T20:12:04,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73. as already flushing 2024-11-18T20:12:04,630 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73. 2024-11-18T20:12:04,630 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 326884d1d40acf488e8bd50aea838c73, NAME => 'TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:12:04,630 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 326884d1d40acf488e8bd50aea838c73, NAME => 'TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:12:04,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41277 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 326884d1d40acf488e8bd50aea838c73, NAME => 'TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 326884d1d40acf488e8bd50aea838c73, NAME => 'TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:12:04,644 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/C/04b4ce1131474dbab3d596c8d7fff1fb, entries=100, sequenceid=12, filesize=9.4 K 2024-11-18T20:12:04,659 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 326884d1d40acf488e8bd50aea838c73 in 550ms, sequenceid=12, compaction requested=false 2024-11-18T20:12:04,660 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-18T20:12:04,663 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 326884d1d40acf488e8bd50aea838c73: 2024-11-18T20:12:04,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-18T20:12:04,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(8855): Flush requested on 326884d1d40acf488e8bd50aea838c73 2024-11-18T20:12:04,668 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-18T20:12:04,669 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 326884d1d40acf488e8bd50aea838c73 3/3 column families, dataSize=161.02 KB heapSize=422.63 KB 2024-11-18T20:12:04,701 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:04,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33028 deadline: 1731960784686, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:04,704 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:04,705 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:04,705 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:12:04,703 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:04,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33070 deadline: 1731960784690, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:04,711 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:04,711 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:04,712 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:12:04,712 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:04,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33046 deadline: 1731960784693, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:04,713 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:04,713 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:04,713 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:12:04,713 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:04,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33056 deadline: 1731960784704, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:04,715 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:04,715 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:04,715 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:12:04,715 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:04,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33072 deadline: 1731960784705, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:04,717 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:04,717 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:04,717 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:12:04,724 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/.tmp/A/fc0a6baebd6d4bf9b155a74d4878d413 is 50, key is test_row_0/A:col10/1731960724282/Put/seqid=0 2024-11-18T20:12:04,764 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073741840_1016 (size=16681) 2024-11-18T20:12:04,797 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35963 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-18T20:12:04,797 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73. 2024-11-18T20:12:04,797 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73. as already flushing 2024-11-18T20:12:04,797 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73. 2024-11-18T20:12:04,797 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 326884d1d40acf488e8bd50aea838c73, NAME => 'TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:12:04,798 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 326884d1d40acf488e8bd50aea838c73, NAME => 'TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:12:04,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41277 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 326884d1d40acf488e8bd50aea838c73, NAME => 'TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 326884d1d40acf488e8bd50aea838c73, NAME => 'TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:12:04,816 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:04,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33028 deadline: 1731960784815, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:04,821 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:04,821 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:04,821 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:12:04,827 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:04,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33056 deadline: 1731960784826, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:04,828 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:04,828 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:04,828 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:12:04,830 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:04,830 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:04,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33072 deadline: 1731960784829, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:04,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33046 deadline: 1731960784829, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:04,831 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:04,831 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:04,832 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:04,832 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:04,832 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:12:04,832 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:12:04,832 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:04,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33070 deadline: 1731960784829, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:04,833 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:04,833 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:04,833 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:12:04,958 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35963 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-18T20:12:04,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73. 2024-11-18T20:12:04,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73. as already flushing 2024-11-18T20:12:04,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73. 2024-11-18T20:12:04,959 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 326884d1d40acf488e8bd50aea838c73, NAME => 'TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:12:04,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 326884d1d40acf488e8bd50aea838c73, NAME => 'TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:12:04,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41277 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 326884d1d40acf488e8bd50aea838c73, NAME => 'TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 326884d1d40acf488e8bd50aea838c73, NAME => 'TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:12:05,025 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:05,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33028 deadline: 1731960785024, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:05,033 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:05,033 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:05,033 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:12:05,040 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:05,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33056 deadline: 1731960785034, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:05,041 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:05,041 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:05,041 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:12:05,046 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:05,046 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:05,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33046 deadline: 1731960785044, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:05,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33072 deadline: 1731960785043, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:05,048 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:05,048 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:05,048 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:05,048 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:05,048 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:05,048 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:12:05,048 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:12:05,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33070 deadline: 1731960785044, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:05,049 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:05,049 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:05,049 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:12:05,116 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35963 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-18T20:12:05,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73. 2024-11-18T20:12:05,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73. as already flushing 2024-11-18T20:12:05,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73. 2024-11-18T20:12:05,117 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 326884d1d40acf488e8bd50aea838c73, NAME => 'TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:12:05,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 326884d1d40acf488e8bd50aea838c73, NAME => 'TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:12:05,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41277 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 326884d1d40acf488e8bd50aea838c73, NAME => 'TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 326884d1d40acf488e8bd50aea838c73, NAME => 'TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:12:05,166 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=39 (bloomFilter=true), to=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/.tmp/A/fc0a6baebd6d4bf9b155a74d4878d413 2024-11-18T20:12:05,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-18T20:12:05,197 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/.tmp/B/cd9e75d138ad4dcc8e1d330154ce4518 is 50, key is test_row_0/B:col10/1731960724282/Put/seqid=0 2024-11-18T20:12:05,243 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073741841_1017 (size=12001) 2024-11-18T20:12:05,273 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35963 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-18T20:12:05,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73. 2024-11-18T20:12:05,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73. as already flushing 2024-11-18T20:12:05,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73. 2024-11-18T20:12:05,273 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 326884d1d40acf488e8bd50aea838c73, NAME => 'TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:12:05,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 326884d1d40acf488e8bd50aea838c73, NAME => 'TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:12:05,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41277 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 326884d1d40acf488e8bd50aea838c73, NAME => 'TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 326884d1d40acf488e8bd50aea838c73, NAME => 'TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:12:05,344 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:05,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33028 deadline: 1731960785343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:05,346 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:05,346 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:05,346 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:12:05,358 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:05,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33056 deadline: 1731960785355, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:05,359 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:05,360 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:05,360 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:12:05,361 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:05,361 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:05,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33046 deadline: 1731960785355, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:05,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33072 deadline: 1731960785355, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:05,362 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:05,362 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:05,362 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:05,362 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:05,362 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:12:05,362 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:12:05,366 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:05,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33070 deadline: 1731960785364, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:05,367 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:05,368 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:05,368 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:12:05,429 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35963 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-18T20:12:05,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73. 2024-11-18T20:12:05,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73. as already flushing 2024-11-18T20:12:05,430 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73. 2024-11-18T20:12:05,430 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 326884d1d40acf488e8bd50aea838c73, NAME => 'TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:12:05,430 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 326884d1d40acf488e8bd50aea838c73, NAME => 'TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:12:05,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41277 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 326884d1d40acf488e8bd50aea838c73, NAME => 'TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 326884d1d40acf488e8bd50aea838c73, NAME => 'TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:12:05,587 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35963 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-18T20:12:05,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73. 2024-11-18T20:12:05,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73. as already flushing 2024-11-18T20:12:05,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73. 2024-11-18T20:12:05,587 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 326884d1d40acf488e8bd50aea838c73, NAME => 'TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:12:05,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 326884d1d40acf488e8bd50aea838c73, NAME => 'TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:12:05,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41277 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 326884d1d40acf488e8bd50aea838c73, NAME => 'TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 326884d1d40acf488e8bd50aea838c73, NAME => 'TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:12:05,645 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=39 (bloomFilter=true), to=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/.tmp/B/cd9e75d138ad4dcc8e1d330154ce4518 2024-11-18T20:12:05,703 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/.tmp/C/df1f4b436c404217ace3b57d8b57be04 is 50, key is test_row_0/C:col10/1731960724282/Put/seqid=0 2024-11-18T20:12:05,737 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073741842_1018 (size=12001) 2024-11-18T20:12:05,742 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35963 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-18T20:12:05,742 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73. 2024-11-18T20:12:05,743 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=39 (bloomFilter=true), to=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/.tmp/C/df1f4b436c404217ace3b57d8b57be04 2024-11-18T20:12:05,743 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73. as already flushing 2024-11-18T20:12:05,743 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73. 2024-11-18T20:12:05,743 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 326884d1d40acf488e8bd50aea838c73, NAME => 'TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:12:05,744 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 326884d1d40acf488e8bd50aea838c73, NAME => 'TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:12:05,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41277 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 326884d1d40acf488e8bd50aea838c73, NAME => 'TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 326884d1d40acf488e8bd50aea838c73, NAME => 'TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:12:05,765 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/.tmp/A/fc0a6baebd6d4bf9b155a74d4878d413 as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/A/fc0a6baebd6d4bf9b155a74d4878d413 2024-11-18T20:12:05,786 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/A/fc0a6baebd6d4bf9b155a74d4878d413, entries=250, sequenceid=39, filesize=16.3 K 2024-11-18T20:12:05,789 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/.tmp/B/cd9e75d138ad4dcc8e1d330154ce4518 as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/B/cd9e75d138ad4dcc8e1d330154ce4518 2024-11-18T20:12:05,803 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/B/cd9e75d138ad4dcc8e1d330154ce4518, entries=150, sequenceid=39, filesize=11.7 K 2024-11-18T20:12:05,806 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/.tmp/C/df1f4b436c404217ace3b57d8b57be04 as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/C/df1f4b436c404217ace3b57d8b57be04 2024-11-18T20:12:05,841 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/C/df1f4b436c404217ace3b57d8b57be04, entries=150, sequenceid=39, filesize=11.7 K 2024-11-18T20:12:05,843 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~422.58 KB/432720, currentSize=40.25 KB/41220 for 326884d1d40acf488e8bd50aea838c73 in 1175ms, sequenceid=39, compaction requested=false 2024-11-18T20:12:05,843 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 326884d1d40acf488e8bd50aea838c73: 2024-11-18T20:12:05,873 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-18T20:12:05,874 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 326884d1d40acf488e8bd50aea838c73 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-18T20:12:05,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(8855): Flush requested on 326884d1d40acf488e8bd50aea838c73 2024-11-18T20:12:05,903 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35963 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-18T20:12:05,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73. 2024-11-18T20:12:05,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73. as already flushing 2024-11-18T20:12:05,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73. 2024-11-18T20:12:05,903 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 326884d1d40acf488e8bd50aea838c73, NAME => 'TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:12:05,904 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 326884d1d40acf488e8bd50aea838c73, NAME => 'TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:12:05,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41277 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 326884d1d40acf488e8bd50aea838c73, NAME => 'TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 326884d1d40acf488e8bd50aea838c73, NAME => 'TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:12:05,908 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/.tmp/A/a5d33b2a90f449abb085753542d95840 is 50, key is test_row_0/A:col10/1731960725866/Put/seqid=0 2024-11-18T20:12:05,945 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073741843_1019 (size=14341) 2024-11-18T20:12:05,946 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/.tmp/A/a5d33b2a90f449abb085753542d95840 2024-11-18T20:12:05,972 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/.tmp/B/bab568e8a5074950986327999f0c32ae is 50, key is test_row_0/B:col10/1731960725866/Put/seqid=0 2024-11-18T20:12:06,000 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:06,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33046 deadline: 1731960785986, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:06,001 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:06,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33028 deadline: 1731960785986, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:06,005 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:06,005 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:06,006 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:12:06,006 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:06,006 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:06,006 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:06,006 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:12:06,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33070 deadline: 1731960785990, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:06,006 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:06,007 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073741844_1020 (size=12001) 2024-11-18T20:12:06,007 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:06,007 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:12:06,007 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:06,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33072 deadline: 1731960785990, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:06,007 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:06,008 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:06,008 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:12:06,014 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:06,014 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/.tmp/B/bab568e8a5074950986327999f0c32ae 2024-11-18T20:12:06,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33056 deadline: 1731960786000, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:06,015 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:06,015 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:06,015 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:12:06,050 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/.tmp/C/5bba0283845d4a62835670bf54969f5d is 50, key is test_row_0/C:col10/1731960725866/Put/seqid=0 2024-11-18T20:12:06,065 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073741845_1021 (size=12001) 2024-11-18T20:12:06,068 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35963 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-18T20:12:06,068 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73. 2024-11-18T20:12:06,068 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73. as already flushing 2024-11-18T20:12:06,069 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73. 2024-11-18T20:12:06,069 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 326884d1d40acf488e8bd50aea838c73, NAME => 'TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:12:06,069 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/.tmp/C/5bba0283845d4a62835670bf54969f5d 2024-11-18T20:12:06,069 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 326884d1d40acf488e8bd50aea838c73, NAME => 'TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:12:06,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41277 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 326884d1d40acf488e8bd50aea838c73, NAME => 'TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 326884d1d40acf488e8bd50aea838c73, NAME => 'TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:12:06,088 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/.tmp/A/a5d33b2a90f449abb085753542d95840 as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/A/a5d33b2a90f449abb085753542d95840 2024-11-18T20:12:06,111 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/A/a5d33b2a90f449abb085753542d95840, entries=200, sequenceid=50, filesize=14.0 K 2024-11-18T20:12:06,114 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/.tmp/B/bab568e8a5074950986327999f0c32ae as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/B/bab568e8a5074950986327999f0c32ae 2024-11-18T20:12:06,122 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:06,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33028 deadline: 1731960786114, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:06,123 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:06,123 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:06,124 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:12:06,126 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:06,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33046 deadline: 1731960786114, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:06,127 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:06,127 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:06,127 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:12:06,127 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:06,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33070 deadline: 1731960786114, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:06,128 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:06,128 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:06,128 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:12:06,128 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:06,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33072 deadline: 1731960786115, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:06,129 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:06,129 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:06,129 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:12:06,133 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/B/bab568e8a5074950986327999f0c32ae, entries=150, sequenceid=50, filesize=11.7 K 2024-11-18T20:12:06,133 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:06,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33056 deadline: 1731960786125, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:06,135 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:06,135 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:06,135 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:12:06,138 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/.tmp/C/5bba0283845d4a62835670bf54969f5d as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/C/5bba0283845d4a62835670bf54969f5d 2024-11-18T20:12:06,159 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/C/5bba0283845d4a62835670bf54969f5d, entries=150, sequenceid=50, filesize=11.7 K 2024-11-18T20:12:06,162 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 326884d1d40acf488e8bd50aea838c73 in 288ms, sequenceid=50, compaction requested=true 2024-11-18T20:12:06,162 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 326884d1d40acf488e8bd50aea838c73: 2024-11-18T20:12:06,169 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 326884d1d40acf488e8bd50aea838c73:A, priority=-2147483648, current under compaction store size is 1 2024-11-18T20:12:06,173 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:12:06,173 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 326884d1d40acf488e8bd50aea838c73:B, priority=-2147483648, current under compaction store size is 2 2024-11-18T20:12:06,173 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:12:06,173 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 326884d1d40acf488e8bd50aea838c73:C, priority=-2147483648, current under compaction store size is 3 2024-11-18T20:12:06,173 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T20:12:06,176 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T20:12:06,176 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T20:12:06,180 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40679 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T20:12:06,180 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33659 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T20:12:06,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-18T20:12:06,184 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HStore(1541): 326884d1d40acf488e8bd50aea838c73/B is initiating minor compaction (all files) 2024-11-18T20:12:06,185 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 326884d1d40acf488e8bd50aea838c73/B in TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73. 2024-11-18T20:12:06,185 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HStore(1541): 326884d1d40acf488e8bd50aea838c73/A is initiating minor compaction (all files) 2024-11-18T20:12:06,185 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 326884d1d40acf488e8bd50aea838c73/A in TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73. 2024-11-18T20:12:06,185 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/A/124683c738434fc8a0d3b04126c95f32, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/A/fc0a6baebd6d4bf9b155a74d4878d413, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/A/a5d33b2a90f449abb085753542d95840] into tmpdir=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/.tmp, totalSize=39.7 K 2024-11-18T20:12:06,185 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/B/bbffcc63619d435293ea9b31e14d1e5b, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/B/cd9e75d138ad4dcc8e1d330154ce4518, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/B/bab568e8a5074950986327999f0c32ae] into tmpdir=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/.tmp, totalSize=32.9 K 2024-11-18T20:12:06,188 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.Compactor(225): Compacting bbffcc63619d435293ea9b31e14d1e5b, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1731960724081 2024-11-18T20:12:06,188 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] compactions.Compactor(225): Compacting 124683c738434fc8a0d3b04126c95f32, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1731960724081 2024-11-18T20:12:06,189 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.Compactor(225): Compacting cd9e75d138ad4dcc8e1d330154ce4518, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=39, earliestPutTs=1731960724282 2024-11-18T20:12:06,190 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] compactions.Compactor(225): Compacting fc0a6baebd6d4bf9b155a74d4878d413, keycount=250, bloomtype=ROW, size=16.3 K, encoding=NONE, compression=NONE, seqNum=39, earliestPutTs=1731960724273 2024-11-18T20:12:06,191 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.Compactor(225): Compacting bab568e8a5074950986327999f0c32ae, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1731960724670 2024-11-18T20:12:06,196 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] compactions.Compactor(225): Compacting a5d33b2a90f449abb085753542d95840, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1731960724670 2024-11-18T20:12:06,225 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35963 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-18T20:12:06,225 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73. 2024-11-18T20:12:06,226 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2902): Flushing 326884d1d40acf488e8bd50aea838c73 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-18T20:12:06,239 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 326884d1d40acf488e8bd50aea838c73#A#compaction#10 average throughput is 0.60 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:12:06,240 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/.tmp/A/cc1bcccd02b64d9ea6fcbc96155bafdf is 50, key is test_row_0/A:col10/1731960725866/Put/seqid=0 2024-11-18T20:12:06,245 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 326884d1d40acf488e8bd50aea838c73#B#compaction#9 average throughput is 0.94 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:12:06,246 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/.tmp/B/44342c347d4d4d61acc1558c507ecacc is 50, key is test_row_0/B:col10/1731960725866/Put/seqid=0 2024-11-18T20:12:06,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/.tmp/A/058f1958519b4d97b9c8fdcfe4ace58e is 50, key is test_row_0/A:col10/1731960725997/Put/seqid=0 2024-11-18T20:12:06,273 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073741846_1022 (size=12104) 2024-11-18T20:12:06,288 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073741847_1023 (size=12104) 2024-11-18T20:12:06,295 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073741848_1024 (size=12001) 2024-11-18T20:12:06,296 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=75 (bloomFilter=true), to=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/.tmp/A/058f1958519b4d97b9c8fdcfe4ace58e 2024-11-18T20:12:06,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/.tmp/B/5e38cff1b0104ee7b10b42cd81c8b66f is 50, key is test_row_0/B:col10/1731960725997/Put/seqid=0 2024-11-18T20:12:06,342 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73. as already flushing 2024-11-18T20:12:06,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(8855): Flush requested on 326884d1d40acf488e8bd50aea838c73 2024-11-18T20:12:06,351 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073741849_1025 (size=12001) 2024-11-18T20:12:06,354 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=75 (bloomFilter=true), to=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/.tmp/B/5e38cff1b0104ee7b10b42cd81c8b66f 2024-11-18T20:12:06,371 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:06,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33072 deadline: 1731960786356, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:06,372 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:06,372 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:06,372 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:12:06,377 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/.tmp/C/0eb22e3ba7fe404ab6a8e4b81c4d7ffe is 50, key is test_row_0/C:col10/1731960725997/Put/seqid=0 2024-11-18T20:12:06,381 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:06,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33028 deadline: 1731960786364, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:06,382 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:06,382 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:06,382 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:12:06,383 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:06,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33056 deadline: 1731960786366, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:06,385 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:06,385 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:06,385 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:06,385 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:12:06,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33070 deadline: 1731960786367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:06,386 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:06,386 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:06,386 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:12:06,386 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:06,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33046 deadline: 1731960786371, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:06,387 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:06,387 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:06,387 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:12:06,415 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073741850_1026 (size=12001) 2024-11-18T20:12:06,416 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=75 (bloomFilter=true), to=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/.tmp/C/0eb22e3ba7fe404ab6a8e4b81c4d7ffe 2024-11-18T20:12:06,431 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/.tmp/A/058f1958519b4d97b9c8fdcfe4ace58e as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/A/058f1958519b4d97b9c8fdcfe4ace58e 2024-11-18T20:12:06,451 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/A/058f1958519b4d97b9c8fdcfe4ace58e, entries=150, sequenceid=75, filesize=11.7 K 2024-11-18T20:12:06,458 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/.tmp/B/5e38cff1b0104ee7b10b42cd81c8b66f as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/B/5e38cff1b0104ee7b10b42cd81c8b66f 2024-11-18T20:12:06,472 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/B/5e38cff1b0104ee7b10b42cd81c8b66f, entries=150, sequenceid=75, filesize=11.7 K 2024-11-18T20:12:06,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/.tmp/C/0eb22e3ba7fe404ab6a8e4b81c4d7ffe as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/C/0eb22e3ba7fe404ab6a8e4b81c4d7ffe 2024-11-18T20:12:06,486 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/C/0eb22e3ba7fe404ab6a8e4b81c4d7ffe, entries=150, sequenceid=75, filesize=11.7 K 2024-11-18T20:12:06,489 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 326884d1d40acf488e8bd50aea838c73 in 264ms, sequenceid=75, compaction requested=true 2024-11-18T20:12:06,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2603): Flush status journal for 326884d1d40acf488e8bd50aea838c73: 2024-11-18T20:12:06,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73. 2024-11-18T20:12:06,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=8 2024-11-18T20:12:06,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41277 {}] master.HMaster(4169): Remote procedure done, pid=8 2024-11-18T20:12:06,498 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-18T20:12:06,498 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 326884d1d40acf488e8bd50aea838c73 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-18T20:12:06,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(8855): Flush requested on 326884d1d40acf488e8bd50aea838c73 2024-11-18T20:12:06,501 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-18T20:12:06,502 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.4650 sec 2024-11-18T20:12:06,512 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees in 2.4920 sec 2024-11-18T20:12:06,521 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/.tmp/A/fbeb1d89f58c4ef59f662a14bd2f04c1 is 50, key is test_row_0/A:col10/1731960726491/Put/seqid=0 2024-11-18T20:12:06,551 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073741851_1027 (size=14341) 2024-11-18T20:12:06,567 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:06,568 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:06,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33070 deadline: 1731960786555, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:06,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33046 deadline: 1731960786557, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:06,569 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:06,569 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:06,569 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:06,569 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:06,569 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:12:06,569 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:06,569 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:12:06,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33056 deadline: 1731960786565, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:06,570 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:06,570 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:06,570 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:12:06,570 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:06,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33072 deadline: 1731960786566, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:06,570 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:06,571 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:06,570 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:06,571 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:12:06,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33028 deadline: 1731960786568, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:06,571 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:06,571 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:06,571 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:12:06,675 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:06,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33046 deadline: 1731960786674, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:06,676 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:06,677 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:06,677 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:12:06,677 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:06,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33056 deadline: 1731960786675, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:06,678 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:06,678 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:06,678 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:06,679 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:12:06,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33072 deadline: 1731960786675, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:06,679 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:06,679 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:06,679 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:12:06,679 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:06,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33070 deadline: 1731960786676, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:06,682 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:06,682 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:06,682 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:12:06,684 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:06,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33028 deadline: 1731960786684, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:06,685 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:06,686 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:06,686 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:12:06,706 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/.tmp/B/44342c347d4d4d61acc1558c507ecacc as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/B/44342c347d4d4d61acc1558c507ecacc 2024-11-18T20:12:06,711 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/.tmp/A/cc1bcccd02b64d9ea6fcbc96155bafdf as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/A/cc1bcccd02b64d9ea6fcbc96155bafdf 2024-11-18T20:12:06,738 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 326884d1d40acf488e8bd50aea838c73/A of 326884d1d40acf488e8bd50aea838c73 into cc1bcccd02b64d9ea6fcbc96155bafdf(size=11.8 K), total size for store is 23.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:12:06,738 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 326884d1d40acf488e8bd50aea838c73/B of 326884d1d40acf488e8bd50aea838c73 into 44342c347d4d4d61acc1558c507ecacc(size=11.8 K), total size for store is 23.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:12:06,738 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 326884d1d40acf488e8bd50aea838c73: 2024-11-18T20:12:06,738 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 326884d1d40acf488e8bd50aea838c73: 2024-11-18T20:12:06,741 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., storeName=326884d1d40acf488e8bd50aea838c73/A, priority=13, startTime=1731960726164; duration=0sec 2024-11-18T20:12:06,741 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., storeName=326884d1d40acf488e8bd50aea838c73/B, priority=13, startTime=1731960726173; duration=0sec 2024-11-18T20:12:06,742 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T20:12:06,742 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T20:12:06,742 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 326884d1d40acf488e8bd50aea838c73:A 2024-11-18T20:12:06,742 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 326884d1d40acf488e8bd50aea838c73:B 2024-11-18T20:12:06,742 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-18T20:12:06,745 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 45660 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-18T20:12:06,745 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HStore(1541): 326884d1d40acf488e8bd50aea838c73/C is initiating minor compaction (all files) 2024-11-18T20:12:06,745 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 326884d1d40acf488e8bd50aea838c73/C in TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73. 2024-11-18T20:12:06,745 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/C/04b4ce1131474dbab3d596c8d7fff1fb, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/C/df1f4b436c404217ace3b57d8b57be04, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/C/5bba0283845d4a62835670bf54969f5d, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/C/0eb22e3ba7fe404ab6a8e4b81c4d7ffe] into tmpdir=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/.tmp, totalSize=44.6 K 2024-11-18T20:12:06,747 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.Compactor(225): Compacting 04b4ce1131474dbab3d596c8d7fff1fb, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1731960724081 2024-11-18T20:12:06,748 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.Compactor(225): Compacting df1f4b436c404217ace3b57d8b57be04, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=39, earliestPutTs=1731960724282 2024-11-18T20:12:06,749 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.Compactor(225): Compacting 5bba0283845d4a62835670bf54969f5d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1731960724670 2024-11-18T20:12:06,750 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.Compactor(225): Compacting 0eb22e3ba7fe404ab6a8e4b81c4d7ffe, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=75, earliestPutTs=1731960725967 2024-11-18T20:12:06,784 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 326884d1d40acf488e8bd50aea838c73#C#compaction#15 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:12:06,785 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/.tmp/C/e2c2956f4098444983416b57cd15b95d is 50, key is test_row_0/C:col10/1731960725997/Put/seqid=0 2024-11-18T20:12:06,829 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073741852_1028 (size=12139) 2024-11-18T20:12:06,849 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/.tmp/C/e2c2956f4098444983416b57cd15b95d as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/C/e2c2956f4098444983416b57cd15b95d 2024-11-18T20:12:06,865 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 326884d1d40acf488e8bd50aea838c73/C of 326884d1d40acf488e8bd50aea838c73 into e2c2956f4098444983416b57cd15b95d(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:12:06,865 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 326884d1d40acf488e8bd50aea838c73: 2024-11-18T20:12:06,866 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., storeName=326884d1d40acf488e8bd50aea838c73/C, priority=12, startTime=1731960726173; duration=0sec 2024-11-18T20:12:06,866 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:12:06,866 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 326884d1d40acf488e8bd50aea838c73:C 2024-11-18T20:12:06,874 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-18T20:12:06,890 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:06,890 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:06,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33072 deadline: 1731960786885, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:06,891 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:06,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33056 deadline: 1731960786885, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:06,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33046 deadline: 1731960786885, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:06,891 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:06,891 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:06,891 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:12:06,891 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:06,892 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:06,892 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:06,892 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:06,892 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:12:06,892 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:12:06,898 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:06,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33070 deadline: 1731960786894, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:06,900 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:06,900 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:06,900 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:06,900 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:12:06,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33028 deadline: 1731960786894, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:06,901 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:06,901 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:06,901 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:12:06,957 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=87 (bloomFilter=true), to=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/.tmp/A/fbeb1d89f58c4ef59f662a14bd2f04c1 2024-11-18T20:12:06,992 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/.tmp/B/9d617b6b7db14c03a90592af7b61f17f is 50, key is test_row_0/B:col10/1731960726491/Put/seqid=0 2024-11-18T20:12:07,003 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-18T20:12:07,026 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073741853_1029 (size=12001) 2024-11-18T20:12:07,031 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=87 (bloomFilter=true), to=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/.tmp/B/9d617b6b7db14c03a90592af7b61f17f 2024-11-18T20:12:07,052 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/.tmp/C/afebc169e18c46c9919d3ee3b05ec557 is 50, key is test_row_0/C:col10/1731960726491/Put/seqid=0 2024-11-18T20:12:07,059 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073741854_1030 (size=12001) 2024-11-18T20:12:07,206 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:07,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33056 deadline: 1731960787204, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:07,207 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:07,207 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:07,207 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:07,207 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:12:07,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33046 deadline: 1731960787204, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:07,208 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:07,208 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:07,208 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:07,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33028 deadline: 1731960787204, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:07,208 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:07,208 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:12:07,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33072 deadline: 1731960787204, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:07,209 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:07,209 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:07,209 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:12:07,209 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:07,209 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:07,209 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:12:07,214 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:07,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33070 deadline: 1731960787213, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:07,218 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:07,218 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:07,218 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:12:07,460 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=87 (bloomFilter=true), to=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/.tmp/C/afebc169e18c46c9919d3ee3b05ec557 2024-11-18T20:12:07,472 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/.tmp/A/fbeb1d89f58c4ef59f662a14bd2f04c1 as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/A/fbeb1d89f58c4ef59f662a14bd2f04c1 2024-11-18T20:12:07,484 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/A/fbeb1d89f58c4ef59f662a14bd2f04c1, entries=200, sequenceid=87, filesize=14.0 K 2024-11-18T20:12:07,492 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/.tmp/B/9d617b6b7db14c03a90592af7b61f17f as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/B/9d617b6b7db14c03a90592af7b61f17f 2024-11-18T20:12:07,506 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/B/9d617b6b7db14c03a90592af7b61f17f, entries=150, sequenceid=87, filesize=11.7 K 2024-11-18T20:12:07,508 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/.tmp/C/afebc169e18c46c9919d3ee3b05ec557 as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/C/afebc169e18c46c9919d3ee3b05ec557 2024-11-18T20:12:07,525 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/C/afebc169e18c46c9919d3ee3b05ec557, entries=150, sequenceid=87, filesize=11.7 K 2024-11-18T20:12:07,529 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 326884d1d40acf488e8bd50aea838c73 in 1031ms, sequenceid=87, compaction requested=true 2024-11-18T20:12:07,531 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 326884d1d40acf488e8bd50aea838c73: 2024-11-18T20:12:07,531 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 326884d1d40acf488e8bd50aea838c73:A, priority=-2147483648, current under compaction store size is 1 2024-11-18T20:12:07,531 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T20:12:07,531 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:12:07,531 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T20:12:07,534 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T20:12:07,535 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HStore(1541): 326884d1d40acf488e8bd50aea838c73/B is initiating minor compaction (all files) 2024-11-18T20:12:07,535 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 326884d1d40acf488e8bd50aea838c73/B in TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73. 2024-11-18T20:12:07,535 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/B/44342c347d4d4d61acc1558c507ecacc, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/B/5e38cff1b0104ee7b10b42cd81c8b66f, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/B/9d617b6b7db14c03a90592af7b61f17f] into tmpdir=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/.tmp, totalSize=35.3 K 2024-11-18T20:12:07,535 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38446 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T20:12:07,536 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HStore(1541): 326884d1d40acf488e8bd50aea838c73/A is initiating minor compaction (all files) 2024-11-18T20:12:07,536 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 326884d1d40acf488e8bd50aea838c73/A in TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73. 2024-11-18T20:12:07,536 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/A/cc1bcccd02b64d9ea6fcbc96155bafdf, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/A/058f1958519b4d97b9c8fdcfe4ace58e, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/A/fbeb1d89f58c4ef59f662a14bd2f04c1] into tmpdir=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/.tmp, totalSize=37.5 K 2024-11-18T20:12:07,537 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] compactions.Compactor(225): Compacting cc1bcccd02b64d9ea6fcbc96155bafdf, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1731960724670 2024-11-18T20:12:07,538 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.Compactor(225): Compacting 44342c347d4d4d61acc1558c507ecacc, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1731960724670 2024-11-18T20:12:07,539 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] compactions.Compactor(225): Compacting 058f1958519b4d97b9c8fdcfe4ace58e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=75, earliestPutTs=1731960725967 2024-11-18T20:12:07,540 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.Compactor(225): Compacting 5e38cff1b0104ee7b10b42cd81c8b66f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=75, earliestPutTs=1731960725967 2024-11-18T20:12:07,541 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.Compactor(225): Compacting 9d617b6b7db14c03a90592af7b61f17f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=87, earliestPutTs=1731960726360 2024-11-18T20:12:07,541 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] compactions.Compactor(225): Compacting fbeb1d89f58c4ef59f662a14bd2f04c1, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=87, earliestPutTs=1731960726337 2024-11-18T20:12:07,549 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 326884d1d40acf488e8bd50aea838c73:B, priority=-2147483648, current under compaction store size is 2 2024-11-18T20:12:07,549 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:12:07,549 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 326884d1d40acf488e8bd50aea838c73:C, priority=-2147483648, current under compaction store size is 3 2024-11-18T20:12:07,549 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T20:12:07,576 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 326884d1d40acf488e8bd50aea838c73#B#compaction#18 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:12:07,578 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/.tmp/B/adfdd57b05bc4ce09a93f307c548e341 is 50, key is test_row_0/B:col10/1731960726491/Put/seqid=0 2024-11-18T20:12:07,579 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 326884d1d40acf488e8bd50aea838c73#A#compaction#19 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:12:07,579 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/.tmp/A/9de89760a89a4c3ab5ce08f397185ffb is 50, key is test_row_0/A:col10/1731960726491/Put/seqid=0 2024-11-18T20:12:07,600 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073741855_1031 (size=12207) 2024-11-18T20:12:07,604 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073741856_1032 (size=12207) 2024-11-18T20:12:07,617 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/.tmp/B/adfdd57b05bc4ce09a93f307c548e341 as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/B/adfdd57b05bc4ce09a93f307c548e341 2024-11-18T20:12:07,620 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/.tmp/A/9de89760a89a4c3ab5ce08f397185ffb as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/A/9de89760a89a4c3ab5ce08f397185ffb 2024-11-18T20:12:07,633 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 326884d1d40acf488e8bd50aea838c73/B of 326884d1d40acf488e8bd50aea838c73 into adfdd57b05bc4ce09a93f307c548e341(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:12:07,633 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 326884d1d40acf488e8bd50aea838c73: 2024-11-18T20:12:07,633 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., storeName=326884d1d40acf488e8bd50aea838c73/B, priority=13, startTime=1731960727531; duration=0sec 2024-11-18T20:12:07,634 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T20:12:07,634 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 326884d1d40acf488e8bd50aea838c73:B 2024-11-18T20:12:07,634 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-18T20:12:07,636 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-18T20:12:07,637 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-18T20:12:07,637 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73. because compaction request was cancelled 2024-11-18T20:12:07,637 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 326884d1d40acf488e8bd50aea838c73:C 2024-11-18T20:12:07,637 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 326884d1d40acf488e8bd50aea838c73/A of 326884d1d40acf488e8bd50aea838c73 into 9de89760a89a4c3ab5ce08f397185ffb(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:12:07,638 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 326884d1d40acf488e8bd50aea838c73: 2024-11-18T20:12:07,638 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., storeName=326884d1d40acf488e8bd50aea838c73/A, priority=13, startTime=1731960727531; duration=0sec 2024-11-18T20:12:07,638 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:12:07,638 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 326884d1d40acf488e8bd50aea838c73:A 2024-11-18T20:12:07,717 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-18T20:12:07,717 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 326884d1d40acf488e8bd50aea838c73 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-18T20:12:07,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(8855): Flush requested on 326884d1d40acf488e8bd50aea838c73 2024-11-18T20:12:07,732 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/.tmp/A/978e5753f01b4e7eb96e9bc30d36ac1c is 50, key is test_row_0/A:col10/1731960726564/Put/seqid=0 2024-11-18T20:12:07,747 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:07,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33046 deadline: 1731960787737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:07,748 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:07,748 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:07,748 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:07,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33070 deadline: 1731960787740, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:07,748 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:12:07,749 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:07,749 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:07,749 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:12:07,753 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:07,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33072 deadline: 1731960787743, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:07,754 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:07,754 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:07,755 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:12:07,756 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:07,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33056 deadline: 1731960787748, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:07,757 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:07,757 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:07,757 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:07,757 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:12:07,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33028 deadline: 1731960787749, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:07,758 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:07,758 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:07,758 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:12:07,767 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073741857_1033 (size=14341) 2024-11-18T20:12:07,768 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=117 (bloomFilter=true), to=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/.tmp/A/978e5753f01b4e7eb96e9bc30d36ac1c 2024-11-18T20:12:07,791 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/.tmp/B/ea3e4562667b4f709a4167a7ed94421d is 50, key is test_row_0/B:col10/1731960726564/Put/seqid=0 2024-11-18T20:12:07,814 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073741858_1034 (size=12001) 2024-11-18T20:12:07,818 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=117 (bloomFilter=true), to=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/.tmp/B/ea3e4562667b4f709a4167a7ed94421d 2024-11-18T20:12:07,839 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/.tmp/C/8e03a13d858b4feda1348a2ae904fc6d is 50, key is test_row_0/C:col10/1731960726564/Put/seqid=0 2024-11-18T20:12:07,855 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:07,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33046 deadline: 1731960787854, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:07,856 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:07,856 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:07,856 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:07,856 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:12:07,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33070 deadline: 1731960787854, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:07,857 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:07,857 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:07,857 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:12:07,864 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:07,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33072 deadline: 1731960787863, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:07,865 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:07,865 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:07,865 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:07,865 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:07,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33028 deadline: 1731960787864, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:07,865 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:12:07,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33056 deadline: 1731960787864, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:07,868 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:07,868 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:07,868 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:07,868 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:07,868 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:12:07,868 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:12:07,877 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073741859_1035 (size=12001) 2024-11-18T20:12:07,879 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=117 (bloomFilter=true), to=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/.tmp/C/8e03a13d858b4feda1348a2ae904fc6d 2024-11-18T20:12:07,894 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/.tmp/A/978e5753f01b4e7eb96e9bc30d36ac1c as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/A/978e5753f01b4e7eb96e9bc30d36ac1c 2024-11-18T20:12:07,908 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/A/978e5753f01b4e7eb96e9bc30d36ac1c, entries=200, sequenceid=117, filesize=14.0 K 2024-11-18T20:12:07,909 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/.tmp/B/ea3e4562667b4f709a4167a7ed94421d as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/B/ea3e4562667b4f709a4167a7ed94421d 2024-11-18T20:12:07,921 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/B/ea3e4562667b4f709a4167a7ed94421d, entries=150, sequenceid=117, filesize=11.7 K 2024-11-18T20:12:07,923 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-18T20:12:07,923 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-11-18T20:12:07,925 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-18T20:12:07,925 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-11-18T20:12:07,926 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-11-18T20:12:07,926 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-11-18T20:12:07,927 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-18T20:12:07,927 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-11-18T20:12:07,928 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/.tmp/C/8e03a13d858b4feda1348a2ae904fc6d as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/C/8e03a13d858b4feda1348a2ae904fc6d 2024-11-18T20:12:07,941 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/C/8e03a13d858b4feda1348a2ae904fc6d, entries=150, sequenceid=117, filesize=11.7 K 2024-11-18T20:12:07,942 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=60.38 KB/61830 for 326884d1d40acf488e8bd50aea838c73 in 225ms, sequenceid=117, compaction requested=true 2024-11-18T20:12:07,942 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 326884d1d40acf488e8bd50aea838c73: 2024-11-18T20:12:07,943 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 326884d1d40acf488e8bd50aea838c73:A, priority=-2147483648, current under compaction store size is 1 2024-11-18T20:12:07,943 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-18T20:12:07,943 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:12:07,943 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 326884d1d40acf488e8bd50aea838c73:B, priority=-2147483648, current under compaction store size is 2 2024-11-18T20:12:07,943 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T20:12:07,943 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 326884d1d40acf488e8bd50aea838c73:C, priority=-2147483648, current under compaction store size is 3 2024-11-18T20:12:07,943 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T20:12:07,943 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T20:12:07,948 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-18T20:12:07,948 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-18T20:12:07,949 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73. because compaction request was cancelled 2024-11-18T20:12:07,949 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 326884d1d40acf488e8bd50aea838c73:A 2024-11-18T20:12:07,949 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-18T20:12:07,950 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36141 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T20:12:07,950 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HStore(1541): 326884d1d40acf488e8bd50aea838c73/C is initiating minor compaction (all files) 2024-11-18T20:12:07,950 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 326884d1d40acf488e8bd50aea838c73/C in TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73. 2024-11-18T20:12:07,950 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/C/e2c2956f4098444983416b57cd15b95d, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/C/afebc169e18c46c9919d3ee3b05ec557, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/C/8e03a13d858b4feda1348a2ae904fc6d] into tmpdir=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/.tmp, totalSize=35.3 K 2024-11-18T20:12:07,951 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-18T20:12:07,951 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.Compactor(225): Compacting e2c2956f4098444983416b57cd15b95d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=75, earliestPutTs=1731960725967 2024-11-18T20:12:07,951 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-18T20:12:07,951 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73. because compaction request was cancelled 2024-11-18T20:12:07,951 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 326884d1d40acf488e8bd50aea838c73:B 2024-11-18T20:12:07,952 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.Compactor(225): Compacting afebc169e18c46c9919d3ee3b05ec557, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=87, earliestPutTs=1731960726360 2024-11-18T20:12:07,954 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.Compactor(225): Compacting 8e03a13d858b4feda1348a2ae904fc6d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1731960726534 2024-11-18T20:12:07,969 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 326884d1d40acf488e8bd50aea838c73#C#compaction#23 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:12:07,970 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/.tmp/C/5a28bdaea97745d6a8cecbe63f246849 is 50, key is test_row_0/C:col10/1731960726564/Put/seqid=0 2024-11-18T20:12:07,982 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073741860_1036 (size=12241) 2024-11-18T20:12:08,009 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/.tmp/C/5a28bdaea97745d6a8cecbe63f246849 as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/C/5a28bdaea97745d6a8cecbe63f246849 2024-11-18T20:12:08,025 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 326884d1d40acf488e8bd50aea838c73/C of 326884d1d40acf488e8bd50aea838c73 into 5a28bdaea97745d6a8cecbe63f246849(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:12:08,025 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 326884d1d40acf488e8bd50aea838c73: 2024-11-18T20:12:08,025 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., storeName=326884d1d40acf488e8bd50aea838c73/C, priority=13, startTime=1731960727943; duration=0sec 2024-11-18T20:12:08,026 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:12:08,026 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 326884d1d40acf488e8bd50aea838c73:C 2024-11-18T20:12:08,070 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-18T20:12:08,070 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 326884d1d40acf488e8bd50aea838c73 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-18T20:12:08,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(8855): Flush requested on 326884d1d40acf488e8bd50aea838c73 2024-11-18T20:12:08,099 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/.tmp/A/7d324f29469a4152a7dff5a74672c4ea is 50, key is test_row_0/A:col10/1731960728063/Put/seqid=0 2024-11-18T20:12:08,111 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073741861_1037 (size=12101) 2024-11-18T20:12:08,112 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=132 (bloomFilter=true), to=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/.tmp/A/7d324f29469a4152a7dff5a74672c4ea 2024-11-18T20:12:08,138 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:08,138 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:08,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33070 deadline: 1731960788119, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:08,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33046 deadline: 1731960788122, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:08,139 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:08,139 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:08,139 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:08,139 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:08,139 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:12:08,139 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:12:08,141 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:08,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33072 deadline: 1731960788136, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:08,142 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:08,142 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:08,142 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:12:08,143 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:08,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33056 deadline: 1731960788138, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:08,144 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:08,144 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:08,144 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:12:08,145 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:08,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33028 deadline: 1731960788138, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:08,146 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:08,146 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:08,146 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:12:08,147 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/.tmp/B/67d27c3ba91c44faa70cc786fcac0d7a is 50, key is test_row_0/B:col10/1731960728063/Put/seqid=0 2024-11-18T20:12:08,169 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073741862_1038 (size=12101) 2024-11-18T20:12:08,170 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=132 (bloomFilter=true), to=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/.tmp/B/67d27c3ba91c44faa70cc786fcac0d7a 2024-11-18T20:12:08,185 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/.tmp/C/575a85dc630d4f7fbe04a6f8e64b7874 is 50, key is test_row_0/C:col10/1731960728063/Put/seqid=0 2024-11-18T20:12:08,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-18T20:12:08,203 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-18T20:12:08,204 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41277 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-18T20:12:08,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41277 {}] procedure2.ProcedureExecutor(1139): Stored pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees 2024-11-18T20:12:08,208 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-18T20:12:08,209 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-18T20:12:08,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-18T20:12:08,210 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=10, ppid=9, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-18T20:12:08,213 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073741863_1039 (size=12101) 2024-11-18T20:12:08,223 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=132 (bloomFilter=true), to=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/.tmp/C/575a85dc630d4f7fbe04a6f8e64b7874 2024-11-18T20:12:08,235 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/.tmp/A/7d324f29469a4152a7dff5a74672c4ea as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/A/7d324f29469a4152a7dff5a74672c4ea 2024-11-18T20:12:08,245 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:08,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33046 deadline: 1731960788244, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:08,246 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:08,246 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:08,246 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:08,246 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:12:08,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33070 deadline: 1731960788244, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:08,246 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/A/7d324f29469a4152a7dff5a74672c4ea, entries=150, sequenceid=132, filesize=11.8 K 2024-11-18T20:12:08,247 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:08,247 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:08,247 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:12:08,249 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/.tmp/B/67d27c3ba91c44faa70cc786fcac0d7a as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/B/67d27c3ba91c44faa70cc786fcac0d7a 2024-11-18T20:12:08,257 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/B/67d27c3ba91c44faa70cc786fcac0d7a, entries=150, sequenceid=132, filesize=11.8 K 2024-11-18T20:12:08,259 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/.tmp/C/575a85dc630d4f7fbe04a6f8e64b7874 as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/C/575a85dc630d4f7fbe04a6f8e64b7874 2024-11-18T20:12:08,262 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:08,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33072 deadline: 1731960788253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:08,264 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:08,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33028 deadline: 1731960788253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:08,263 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:08,265 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:08,265 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:08,265 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:08,265 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:08,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33056 deadline: 1731960788253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:08,265 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:12:08,265 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:12:08,266 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:08,267 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:08,267 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:12:08,268 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/C/575a85dc630d4f7fbe04a6f8e64b7874, entries=150, sequenceid=132, filesize=11.8 K 2024-11-18T20:12:08,270 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for 326884d1d40acf488e8bd50aea838c73 in 200ms, sequenceid=132, compaction requested=true 2024-11-18T20:12:08,270 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 326884d1d40acf488e8bd50aea838c73: 2024-11-18T20:12:08,270 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 326884d1d40acf488e8bd50aea838c73:A, priority=-2147483648, current under compaction store size is 1 2024-11-18T20:12:08,271 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T20:12:08,271 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:12:08,271 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T20:12:08,271 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 326884d1d40acf488e8bd50aea838c73:B, priority=-2147483648, current under compaction store size is 2 2024-11-18T20:12:08,271 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:12:08,271 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 326884d1d40acf488e8bd50aea838c73:C, priority=-2147483648, current under compaction store size is 3 2024-11-18T20:12:08,271 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T20:12:08,272 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36309 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T20:12:08,272 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38649 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T20:12:08,272 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HStore(1541): 326884d1d40acf488e8bd50aea838c73/B is initiating minor compaction (all files) 2024-11-18T20:12:08,272 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HStore(1541): 326884d1d40acf488e8bd50aea838c73/A is initiating minor compaction (all files) 2024-11-18T20:12:08,272 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 326884d1d40acf488e8bd50aea838c73/B in TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73. 2024-11-18T20:12:08,273 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/B/adfdd57b05bc4ce09a93f307c548e341, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/B/ea3e4562667b4f709a4167a7ed94421d, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/B/67d27c3ba91c44faa70cc786fcac0d7a] into tmpdir=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/.tmp, totalSize=35.5 K 2024-11-18T20:12:08,273 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 326884d1d40acf488e8bd50aea838c73/A in TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73. 2024-11-18T20:12:08,273 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/A/9de89760a89a4c3ab5ce08f397185ffb, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/A/978e5753f01b4e7eb96e9bc30d36ac1c, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/A/7d324f29469a4152a7dff5a74672c4ea] into tmpdir=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/.tmp, totalSize=37.7 K 2024-11-18T20:12:08,274 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.Compactor(225): Compacting adfdd57b05bc4ce09a93f307c548e341, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=87, earliestPutTs=1731960726360 2024-11-18T20:12:08,274 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9de89760a89a4c3ab5ce08f397185ffb, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=87, earliestPutTs=1731960726360 2024-11-18T20:12:08,275 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] compactions.Compactor(225): Compacting 978e5753f01b4e7eb96e9bc30d36ac1c, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1731960726534 2024-11-18T20:12:08,275 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.Compactor(225): Compacting ea3e4562667b4f709a4167a7ed94421d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1731960726534 2024-11-18T20:12:08,275 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7d324f29469a4152a7dff5a74672c4ea, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1731960727736 2024-11-18T20:12:08,275 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.Compactor(225): Compacting 67d27c3ba91c44faa70cc786fcac0d7a, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1731960727736 2024-11-18T20:12:08,292 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 326884d1d40acf488e8bd50aea838c73#A#compaction#27 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:12:08,293 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/.tmp/A/a4cb193dc4094156908221894e02b71e is 50, key is test_row_0/A:col10/1731960728063/Put/seqid=0 2024-11-18T20:12:08,298 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 326884d1d40acf488e8bd50aea838c73#B#compaction#28 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:12:08,299 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/.tmp/B/8f1b446ff4c740d8b5bed39e5ae5d758 is 50, key is test_row_0/B:col10/1731960728063/Put/seqid=0 2024-11-18T20:12:08,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-18T20:12:08,321 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073741865_1041 (size=12409) 2024-11-18T20:12:08,338 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/.tmp/B/8f1b446ff4c740d8b5bed39e5ae5d758 as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/B/8f1b446ff4c740d8b5bed39e5ae5d758 2024-11-18T20:12:08,349 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073741864_1040 (size=12409) 2024-11-18T20:12:08,357 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 326884d1d40acf488e8bd50aea838c73/B of 326884d1d40acf488e8bd50aea838c73 into 8f1b446ff4c740d8b5bed39e5ae5d758(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:12:08,357 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 326884d1d40acf488e8bd50aea838c73: 2024-11-18T20:12:08,357 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., storeName=326884d1d40acf488e8bd50aea838c73/B, priority=13, startTime=1731960728271; duration=0sec 2024-11-18T20:12:08,357 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T20:12:08,357 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 326884d1d40acf488e8bd50aea838c73:B 2024-11-18T20:12:08,357 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-18T20:12:08,359 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-18T20:12:08,359 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-18T20:12:08,360 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73. because compaction request was cancelled 2024-11-18T20:12:08,360 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 326884d1d40acf488e8bd50aea838c73:C 2024-11-18T20:12:08,364 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35963 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-18T20:12:08,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73. 2024-11-18T20:12:08,364 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2902): Flushing 326884d1d40acf488e8bd50aea838c73 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-18T20:12:08,366 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/.tmp/A/a4cb193dc4094156908221894e02b71e as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/A/a4cb193dc4094156908221894e02b71e 2024-11-18T20:12:08,380 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 326884d1d40acf488e8bd50aea838c73/A of 326884d1d40acf488e8bd50aea838c73 into a4cb193dc4094156908221894e02b71e(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:12:08,380 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 326884d1d40acf488e8bd50aea838c73: 2024-11-18T20:12:08,380 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., storeName=326884d1d40acf488e8bd50aea838c73/A, priority=13, startTime=1731960728270; duration=0sec 2024-11-18T20:12:08,380 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:12:08,380 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 326884d1d40acf488e8bd50aea838c73:A 2024-11-18T20:12:08,383 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/.tmp/A/47e69739427c498aa338823af7293720 is 50, key is test_row_0/A:col10/1731960728132/Put/seqid=0 2024-11-18T20:12:08,431 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073741866_1042 (size=12151) 2024-11-18T20:12:08,432 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=155 (bloomFilter=true), to=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/.tmp/A/47e69739427c498aa338823af7293720 2024-11-18T20:12:08,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/.tmp/B/f0a5d5a160bd4c8badea0af09518bde9 is 50, key is test_row_0/B:col10/1731960728132/Put/seqid=0 2024-11-18T20:12:08,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(8855): Flush requested on 326884d1d40acf488e8bd50aea838c73 2024-11-18T20:12:08,460 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73. as already flushing 2024-11-18T20:12:08,465 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073741867_1043 (size=12151) 2024-11-18T20:12:08,494 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:08,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33028 deadline: 1731960788485, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:08,495 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:08,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33046 deadline: 1731960788487, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:08,495 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:08,495 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:08,495 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:12:08,495 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:08,496 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:08,496 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:08,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33070 deadline: 1731960788490, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:08,496 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:12:08,496 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:08,496 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:08,496 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:12:08,501 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:08,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33056 deadline: 1731960788494, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:08,501 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:08,502 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:08,502 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:08,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33072 deadline: 1731960788495, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:08,502 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:12:08,502 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:08,502 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:08,502 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:12:08,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-18T20:12:08,607 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:08,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33028 deadline: 1731960788603, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:08,609 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:08,609 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:08,609 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:08,609 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:12:08,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33070 deadline: 1731960788604, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:08,610 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:08,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33056 deadline: 1731960788604, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:08,611 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:08,611 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:08,611 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:12:08,611 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:08,611 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:08,611 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:12:08,604 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:08,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33046 deadline: 1731960788603, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:08,618 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:08,618 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:08,618 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:08,618 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:12:08,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33072 deadline: 1731960788613, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:08,619 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:08,619 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:08,619 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:12:08,816 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:08,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33028 deadline: 1731960788815, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:08,817 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:08,818 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:08,818 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:12:08,825 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:08,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33056 deadline: 1731960788824, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:08,827 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:08,827 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:08,827 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:12:08,827 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:08,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33070 deadline: 1731960788824, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:08,828 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:08,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33072 deadline: 1731960788824, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:08,829 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:08,829 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:08,829 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:12:08,830 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:08,830 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:08,830 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:12:08,830 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:08,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33046 deadline: 1731960788824, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:08,831 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:08,831 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:08,831 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:12:08,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-18T20:12:08,867 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=155 (bloomFilter=true), to=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/.tmp/B/f0a5d5a160bd4c8badea0af09518bde9 2024-11-18T20:12:08,894 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/.tmp/C/9f393cdd80ce414790fc9040d5f8fbcc is 50, key is test_row_0/C:col10/1731960728132/Put/seqid=0 2024-11-18T20:12:08,935 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073741868_1044 (size=12151) 2024-11-18T20:12:08,937 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=155 (bloomFilter=true), to=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/.tmp/C/9f393cdd80ce414790fc9040d5f8fbcc 2024-11-18T20:12:08,949 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/.tmp/A/47e69739427c498aa338823af7293720 as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/A/47e69739427c498aa338823af7293720 2024-11-18T20:12:08,961 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/A/47e69739427c498aa338823af7293720, entries=150, sequenceid=155, filesize=11.9 K 2024-11-18T20:12:08,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/.tmp/B/f0a5d5a160bd4c8badea0af09518bde9 as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/B/f0a5d5a160bd4c8badea0af09518bde9 2024-11-18T20:12:08,975 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/B/f0a5d5a160bd4c8badea0af09518bde9, entries=150, sequenceid=155, filesize=11.9 K 2024-11-18T20:12:08,979 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/.tmp/C/9f393cdd80ce414790fc9040d5f8fbcc as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/C/9f393cdd80ce414790fc9040d5f8fbcc 2024-11-18T20:12:09,001 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/C/9f393cdd80ce414790fc9040d5f8fbcc, entries=150, sequenceid=155, filesize=11.9 K 2024-11-18T20:12:09,003 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=73.80 KB/75570 for 326884d1d40acf488e8bd50aea838c73 in 638ms, sequenceid=155, compaction requested=true 2024-11-18T20:12:09,003 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2603): Flush status journal for 326884d1d40acf488e8bd50aea838c73: 2024-11-18T20:12:09,003 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73. 2024-11-18T20:12:09,003 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=10 2024-11-18T20:12:09,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41277 {}] master.HMaster(4169): Remote procedure done, pid=10 2024-11-18T20:12:09,014 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=10, resume processing ppid=9 2024-11-18T20:12:09,014 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 799 msec 2024-11-18T20:12:09,022 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees in 812 msec 2024-11-18T20:12:09,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(8855): Flush requested on 326884d1d40acf488e8bd50aea838c73 2024-11-18T20:12:09,130 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-18T20:12:09,130 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 326884d1d40acf488e8bd50aea838c73 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-11-18T20:12:09,140 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/.tmp/A/dd8f3a6186344a34b25bf0e72add49a4 is 50, key is test_row_0/A:col10/1731960729126/Put/seqid=0 2024-11-18T20:12:09,149 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073741869_1045 (size=14541) 2024-11-18T20:12:09,152 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=171 (bloomFilter=true), to=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/.tmp/A/dd8f3a6186344a34b25bf0e72add49a4 2024-11-18T20:12:09,182 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/.tmp/B/cfc19337f7d1465584f715a6e740d167 is 50, key is test_row_0/B:col10/1731960729126/Put/seqid=0 2024-11-18T20:12:09,207 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073741870_1046 (size=12151) 2024-11-18T20:12:09,237 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:09,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33070 deadline: 1731960789228, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:09,238 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:09,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33056 deadline: 1731960789230, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:09,239 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:09,239 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:09,239 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:09,239 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:12:09,239 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:09,239 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:09,239 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:09,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33072 deadline: 1731960789230, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:09,240 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:12:09,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33028 deadline: 1731960789228, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:09,240 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:09,240 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:09,240 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:09,240 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:12:09,240 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:09,240 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:12:09,240 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:09,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33046 deadline: 1731960789228, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:09,241 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:09,241 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:09,241 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:12:09,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-18T20:12:09,344 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-18T20:12:09,345 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41277 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-18T20:12:09,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41277 {}] procedure2.ProcedureExecutor(1139): Stored pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees 2024-11-18T20:12:09,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-18T20:12:09,348 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-18T20:12:09,349 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:09,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33056 deadline: 1731960789343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:09,350 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:09,350 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:09,350 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:09,350 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:09,350 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:12:09,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33046 deadline: 1731960789344, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:09,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33072 deadline: 1731960789344, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:09,350 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-18T20:12:09,350 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:09,350 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:09,351 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=11, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-18T20:12:09,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33028 deadline: 1731960789344, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:09,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33070 deadline: 1731960789344, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:09,351 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:09,351 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:09,351 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:12:09,351 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:09,351 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:09,352 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:12:09,352 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:09,352 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:09,352 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:12:09,352 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:09,352 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:09,352 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:12:09,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-18T20:12:09,505 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35963 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-18T20:12:09,506 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73. 2024-11-18T20:12:09,506 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73. as already flushing 2024-11-18T20:12:09,506 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73. 2024-11-18T20:12:09,506 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 326884d1d40acf488e8bd50aea838c73, NAME => 'TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:12:09,506 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 326884d1d40acf488e8bd50aea838c73, NAME => 'TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:12:09,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41277 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 326884d1d40acf488e8bd50aea838c73, NAME => 'TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 326884d1d40acf488e8bd50aea838c73, NAME => 'TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:12:09,555 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:09,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33056 deadline: 1731960789553, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:09,556 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:09,556 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:09,556 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:12:09,565 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:09,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33072 deadline: 1731960789563, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:09,566 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:09,566 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:09,566 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:09,566 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:12:09,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33028 deadline: 1731960789564, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:09,566 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:09,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33046 deadline: 1731960789564, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:09,567 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:09,567 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:09,567 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:12:09,567 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:09,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33070 deadline: 1731960789564, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:09,567 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:09,567 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:09,567 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:12:09,567 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:09,568 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:09,568 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:12:09,609 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=171 (bloomFilter=true), to=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/.tmp/B/cfc19337f7d1465584f715a6e740d167 2024-11-18T20:12:09,630 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/.tmp/C/6ee05491a9b94a5c86dee6e8fd3fa393 is 50, key is test_row_0/C:col10/1731960729126/Put/seqid=0 2024-11-18T20:12:09,647 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073741871_1047 (size=12151) 2024-11-18T20:12:09,648 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=171 (bloomFilter=true), to=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/.tmp/C/6ee05491a9b94a5c86dee6e8fd3fa393 2024-11-18T20:12:09,659 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/.tmp/A/dd8f3a6186344a34b25bf0e72add49a4 as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/A/dd8f3a6186344a34b25bf0e72add49a4 2024-11-18T20:12:09,660 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35963 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-18T20:12:09,660 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73. 2024-11-18T20:12:09,660 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73. as already flushing 2024-11-18T20:12:09,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73. 2024-11-18T20:12:09,661 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 326884d1d40acf488e8bd50aea838c73, NAME => 'TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:12:09,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 326884d1d40acf488e8bd50aea838c73, NAME => 'TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:12:09,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41277 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 326884d1d40acf488e8bd50aea838c73, NAME => 'TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 326884d1d40acf488e8bd50aea838c73, NAME => 'TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:12:09,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-18T20:12:09,675 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/A/dd8f3a6186344a34b25bf0e72add49a4, entries=200, sequenceid=171, filesize=14.2 K 2024-11-18T20:12:09,677 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/.tmp/B/cfc19337f7d1465584f715a6e740d167 as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/B/cfc19337f7d1465584f715a6e740d167 2024-11-18T20:12:09,685 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/B/cfc19337f7d1465584f715a6e740d167, entries=150, sequenceid=171, filesize=11.9 K 2024-11-18T20:12:09,687 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/.tmp/C/6ee05491a9b94a5c86dee6e8fd3fa393 as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/C/6ee05491a9b94a5c86dee6e8fd3fa393 2024-11-18T20:12:09,705 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/C/6ee05491a9b94a5c86dee6e8fd3fa393, entries=150, sequenceid=171, filesize=11.9 K 2024-11-18T20:12:09,707 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=120.76 KB/123660 for 326884d1d40acf488e8bd50aea838c73 in 576ms, sequenceid=171, compaction requested=true 2024-11-18T20:12:09,707 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 326884d1d40acf488e8bd50aea838c73: 2024-11-18T20:12:09,707 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T20:12:09,713 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39101 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T20:12:09,713 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HStore(1541): 326884d1d40acf488e8bd50aea838c73/A is initiating minor compaction (all files) 2024-11-18T20:12:09,713 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 326884d1d40acf488e8bd50aea838c73/A in TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73. 2024-11-18T20:12:09,713 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/A/a4cb193dc4094156908221894e02b71e, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/A/47e69739427c498aa338823af7293720, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/A/dd8f3a6186344a34b25bf0e72add49a4] into tmpdir=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/.tmp, totalSize=38.2 K 2024-11-18T20:12:09,714 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] compactions.Compactor(225): Compacting a4cb193dc4094156908221894e02b71e, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1731960727736 2024-11-18T20:12:09,715 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] compactions.Compactor(225): Compacting 47e69739427c498aa338823af7293720, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1731960728118 2024-11-18T20:12:09,716 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] compactions.Compactor(225): Compacting dd8f3a6186344a34b25bf0e72add49a4, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=171, earliestPutTs=1731960728481 2024-11-18T20:12:09,730 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 326884d1d40acf488e8bd50aea838c73:A, priority=-2147483648, current under compaction store size is 1 2024-11-18T20:12:09,732 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:12:09,732 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T20:12:09,734 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36711 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T20:12:09,734 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HStore(1541): 326884d1d40acf488e8bd50aea838c73/B is initiating minor compaction (all files) 2024-11-18T20:12:09,734 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 326884d1d40acf488e8bd50aea838c73/B in TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73. 2024-11-18T20:12:09,734 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/B/8f1b446ff4c740d8b5bed39e5ae5d758, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/B/f0a5d5a160bd4c8badea0af09518bde9, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/B/cfc19337f7d1465584f715a6e740d167] into tmpdir=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/.tmp, totalSize=35.9 K 2024-11-18T20:12:09,735 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.Compactor(225): Compacting 8f1b446ff4c740d8b5bed39e5ae5d758, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1731960727736 2024-11-18T20:12:09,738 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.Compactor(225): Compacting f0a5d5a160bd4c8badea0af09518bde9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1731960728118 2024-11-18T20:12:09,739 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.Compactor(225): Compacting cfc19337f7d1465584f715a6e740d167, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=171, earliestPutTs=1731960728481 2024-11-18T20:12:09,746 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 326884d1d40acf488e8bd50aea838c73#A#compaction#35 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:12:09,747 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/.tmp/A/33812c745aee4ce8bddf200a129b9f9b is 50, key is test_row_0/A:col10/1731960729126/Put/seqid=0 2024-11-18T20:12:09,752 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 326884d1d40acf488e8bd50aea838c73:B, priority=-2147483648, current under compaction store size is 2 2024-11-18T20:12:09,752 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:12:09,753 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 326884d1d40acf488e8bd50aea838c73:C, priority=-2147483648, current under compaction store size is 3 2024-11-18T20:12:09,753 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T20:12:09,773 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 326884d1d40acf488e8bd50aea838c73#B#compaction#36 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:12:09,774 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/.tmp/B/a77f383b179545ec8b284f279bdeb156 is 50, key is test_row_0/B:col10/1731960729126/Put/seqid=0 2024-11-18T20:12:09,785 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073741872_1048 (size=12561) 2024-11-18T20:12:09,795 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/.tmp/A/33812c745aee4ce8bddf200a129b9f9b as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/A/33812c745aee4ce8bddf200a129b9f9b 2024-11-18T20:12:09,809 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 326884d1d40acf488e8bd50aea838c73/A of 326884d1d40acf488e8bd50aea838c73 into 33812c745aee4ce8bddf200a129b9f9b(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:12:09,809 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 326884d1d40acf488e8bd50aea838c73: 2024-11-18T20:12:09,809 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., storeName=326884d1d40acf488e8bd50aea838c73/A, priority=13, startTime=1731960729707; duration=0sec 2024-11-18T20:12:09,810 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T20:12:09,810 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 326884d1d40acf488e8bd50aea838c73:A 2024-11-18T20:12:09,810 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-18T20:12:09,812 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48644 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-18T20:12:09,812 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HStore(1541): 326884d1d40acf488e8bd50aea838c73/C is initiating minor compaction (all files) 2024-11-18T20:12:09,812 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 326884d1d40acf488e8bd50aea838c73/C in TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73. 2024-11-18T20:12:09,812 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/C/5a28bdaea97745d6a8cecbe63f246849, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/C/575a85dc630d4f7fbe04a6f8e64b7874, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/C/9f393cdd80ce414790fc9040d5f8fbcc, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/C/6ee05491a9b94a5c86dee6e8fd3fa393] into tmpdir=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/.tmp, totalSize=47.5 K 2024-11-18T20:12:09,813 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5a28bdaea97745d6a8cecbe63f246849, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1731960726534 2024-11-18T20:12:09,815 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073741873_1049 (size=12561) 2024-11-18T20:12:09,815 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35963 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-18T20:12:09,816 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73. 2024-11-18T20:12:09,816 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2902): Flushing 326884d1d40acf488e8bd50aea838c73 3/3 column families, dataSize=120.76 KB heapSize=317.16 KB 2024-11-18T20:12:09,818 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] compactions.Compactor(225): Compacting 575a85dc630d4f7fbe04a6f8e64b7874, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1731960727736 2024-11-18T20:12:09,820 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9f393cdd80ce414790fc9040d5f8fbcc, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1731960728118 2024-11-18T20:12:09,822 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6ee05491a9b94a5c86dee6e8fd3fa393, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=171, earliestPutTs=1731960728481 2024-11-18T20:12:09,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/.tmp/A/62c2388048994dd2a0162d466fcc1c10 is 50, key is test_row_0/A:col10/1731960729227/Put/seqid=0 2024-11-18T20:12:09,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(8855): Flush requested on 326884d1d40acf488e8bd50aea838c73 2024-11-18T20:12:09,868 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73. as already flushing 2024-11-18T20:12:09,887 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073741874_1050 (size=12151) 2024-11-18T20:12:09,890 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 326884d1d40acf488e8bd50aea838c73#C#compaction#38 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:12:09,891 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/.tmp/C/74f1d8e339d342a9ade550c080960874 is 50, key is test_row_0/C:col10/1731960729126/Put/seqid=0 2024-11-18T20:12:09,909 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:09,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33028 deadline: 1731960789897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:09,910 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:09,910 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:09,910 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:09,910 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:12:09,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33046 deadline: 1731960789900, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:09,910 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:09,911 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:09,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33072 deadline: 1731960789902, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:09,911 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:09,911 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:09,911 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:12:09,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33070 deadline: 1731960789904, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:09,911 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:09,911 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:09,911 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:12:09,911 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:09,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33056 deadline: 1731960789906, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:09,912 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:09,912 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:09,912 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:12:09,912 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:09,912 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:09,913 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:12:09,934 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073741875_1051 (size=12527) 2024-11-18T20:12:09,952 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/.tmp/C/74f1d8e339d342a9ade550c080960874 as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/C/74f1d8e339d342a9ade550c080960874 2024-11-18T20:12:09,966 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 326884d1d40acf488e8bd50aea838c73/C of 326884d1d40acf488e8bd50aea838c73 into 74f1d8e339d342a9ade550c080960874(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:12:09,966 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 326884d1d40acf488e8bd50aea838c73: 2024-11-18T20:12:09,966 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., storeName=326884d1d40acf488e8bd50aea838c73/C, priority=12, startTime=1731960729753; duration=0sec 2024-11-18T20:12:09,966 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:12:09,966 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 326884d1d40acf488e8bd50aea838c73:C 2024-11-18T20:12:09,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-18T20:12:10,017 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:10,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33028 deadline: 1731960790013, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:10,017 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:10,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33046 deadline: 1731960790013, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:10,018 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:10,018 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:10,018 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:12:10,018 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:10,018 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:10,018 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:12:10,018 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:10,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33072 deadline: 1731960790013, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:10,019 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:10,019 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:10,019 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:12:10,028 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:10,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33056 deadline: 1731960790024, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:10,029 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:10,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33070 deadline: 1731960790024, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:10,030 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:10,031 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:10,031 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:12:10,031 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:10,031 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:10,031 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:12:10,226 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/.tmp/B/a77f383b179545ec8b284f279bdeb156 as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/B/a77f383b179545ec8b284f279bdeb156 2024-11-18T20:12:10,230 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:10,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33028 deadline: 1731960790224, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:10,231 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:10,231 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:10,231 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:10,231 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:12:10,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33046 deadline: 1731960790224, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:10,232 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:10,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33072 deadline: 1731960790224, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:10,233 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:10,233 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:10,233 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:10,233 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:10,233 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:12:10,233 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:12:10,234 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:10,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33056 deadline: 1731960790234, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:10,235 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:10,235 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:10,235 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:12:10,244 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:10,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33070 deadline: 1731960790243, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:10,245 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:10,245 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:10,245 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:12:10,246 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 326884d1d40acf488e8bd50aea838c73/B of 326884d1d40acf488e8bd50aea838c73 into a77f383b179545ec8b284f279bdeb156(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:12:10,246 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 326884d1d40acf488e8bd50aea838c73: 2024-11-18T20:12:10,246 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., storeName=326884d1d40acf488e8bd50aea838c73/B, priority=13, startTime=1731960729732; duration=0sec 2024-11-18T20:12:10,247 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:12:10,247 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 326884d1d40acf488e8bd50aea838c73:B 2024-11-18T20:12:10,287 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=193 (bloomFilter=true), to=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/.tmp/A/62c2388048994dd2a0162d466fcc1c10 2024-11-18T20:12:10,313 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/.tmp/B/9763ff43338141c2a7be872384367835 is 50, key is test_row_0/B:col10/1731960729227/Put/seqid=0 2024-11-18T20:12:10,355 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073741876_1052 (size=12151) 2024-11-18T20:12:10,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-18T20:12:10,534 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:10,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33028 deadline: 1731960790533, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:10,535 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:10,535 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:10,535 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:12:10,547 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:10,547 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:10,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33072 deadline: 1731960790543, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:10,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33056 deadline: 1731960790543, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:10,547 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:10,548 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:10,548 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:10,548 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:10,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33046 deadline: 1731960790543, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:10,548 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:10,548 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:12:10,548 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:12:10,548 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:10,548 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:10,548 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:12:10,554 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:10,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33070 deadline: 1731960790553, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:10,557 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:10,557 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:10,557 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:12:10,756 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=193 (bloomFilter=true), to=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/.tmp/B/9763ff43338141c2a7be872384367835 2024-11-18T20:12:10,771 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/.tmp/C/427bbd64ca6947c9b52afa0d85685f9a is 50, key is test_row_0/C:col10/1731960729227/Put/seqid=0 2024-11-18T20:12:10,782 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073741877_1053 (size=12151) 2024-11-18T20:12:10,783 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=193 (bloomFilter=true), to=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/.tmp/C/427bbd64ca6947c9b52afa0d85685f9a 2024-11-18T20:12:10,793 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/.tmp/A/62c2388048994dd2a0162d466fcc1c10 as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/A/62c2388048994dd2a0162d466fcc1c10 2024-11-18T20:12:10,803 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/A/62c2388048994dd2a0162d466fcc1c10, entries=150, sequenceid=193, filesize=11.9 K 2024-11-18T20:12:10,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/.tmp/B/9763ff43338141c2a7be872384367835 as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/B/9763ff43338141c2a7be872384367835 2024-11-18T20:12:10,828 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/B/9763ff43338141c2a7be872384367835, entries=150, sequenceid=193, filesize=11.9 K 2024-11-18T20:12:10,830 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/.tmp/C/427bbd64ca6947c9b52afa0d85685f9a as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/C/427bbd64ca6947c9b52afa0d85685f9a 2024-11-18T20:12:10,843 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/C/427bbd64ca6947c9b52afa0d85685f9a, entries=150, sequenceid=193, filesize=11.9 K 2024-11-18T20:12:10,844 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~317.11 KB/324720, currentSize=80.51 KB/82440 for 326884d1d40acf488e8bd50aea838c73 in 1028ms, sequenceid=193, compaction requested=false 2024-11-18T20:12:10,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2603): Flush status journal for 326884d1d40acf488e8bd50aea838c73: 2024-11-18T20:12:10,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73. 2024-11-18T20:12:10,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=12 2024-11-18T20:12:10,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41277 {}] master.HMaster(4169): Remote procedure done, pid=12 2024-11-18T20:12:10,870 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=11 2024-11-18T20:12:10,870 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5010 sec 2024-11-18T20:12:10,878 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees in 1.5270 sec 2024-11-18T20:12:11,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(8855): Flush requested on 326884d1d40acf488e8bd50aea838c73 2024-11-18T20:12:11,061 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-18T20:12:11,062 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 326884d1d40acf488e8bd50aea838c73 3/3 column families, dataSize=93.93 KB heapSize=246.84 KB 2024-11-18T20:12:11,070 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/.tmp/A/12ef51908c6e4012b11635f0a3bd529b is 50, key is test_row_0/A:col10/1731960729892/Put/seqid=0 2024-11-18T20:12:11,087 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:11,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33028 deadline: 1731960791084, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:11,088 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:11,088 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:11,088 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:12:11,091 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:11,091 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:11,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33046 deadline: 1731960791084, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:11,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33072 deadline: 1731960791085, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:11,092 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:11,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33056 deadline: 1731960791087, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:11,092 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:11,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33070 deadline: 1731960791089, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:11,093 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:11,093 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:11,093 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:12:11,093 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:11,094 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:11,094 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:12:11,094 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:11,094 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:11,094 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:12:11,094 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:11,094 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:11,094 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:12:11,107 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073741878_1054 (size=14541) 2024-11-18T20:12:11,194 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:11,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33028 deadline: 1731960791194, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:11,195 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:11,195 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:11,195 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:12:11,205 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:11,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33070 deadline: 1731960791204, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:11,206 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:11,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33046 deadline: 1731960791205, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:11,206 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:11,206 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:11,206 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:12:11,206 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:11,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33072 deadline: 1731960791205, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:11,206 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:11,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33056 deadline: 1731960791205, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:11,208 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:11,208 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:11,208 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:12:11,208 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:11,209 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:11,209 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:12:11,209 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:11,209 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:11,209 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:12:11,404 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:11,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33028 deadline: 1731960791403, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:11,404 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:11,404 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:11,404 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:12:11,416 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:11,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33070 deadline: 1731960791413, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:11,417 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:11,417 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:11,417 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:12:11,417 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:11,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33056 deadline: 1731960791413, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:11,417 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:11,418 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:11,418 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:12:11,417 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:11,418 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:11,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33046 deadline: 1731960791414, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:11,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33072 deadline: 1731960791414, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:11,418 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:11,418 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:11,418 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:11,418 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:11,418 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:12:11,418 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:12:11,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-18T20:12:11,504 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-18T20:12:11,506 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41277 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-18T20:12:11,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41277 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees 2024-11-18T20:12:11,508 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=214 (bloomFilter=true), to=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/.tmp/A/12ef51908c6e4012b11635f0a3bd529b 2024-11-18T20:12:11,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-18T20:12:11,510 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-18T20:12:11,512 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-18T20:12:11,513 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-18T20:12:11,531 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/.tmp/B/1671a674abe646418de4581d5ac37ac4 is 50, key is test_row_0/B:col10/1731960729892/Put/seqid=0 2024-11-18T20:12:11,563 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073741879_1055 (size=12151) 2024-11-18T20:12:11,564 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=214 (bloomFilter=true), to=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/.tmp/B/1671a674abe646418de4581d5ac37ac4 2024-11-18T20:12:11,578 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/.tmp/C/afe362d8843742188e5c63ff3c9ce7d8 is 50, key is test_row_0/C:col10/1731960729892/Put/seqid=0 2024-11-18T20:12:11,609 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073741880_1056 (size=12151) 2024-11-18T20:12:11,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-18T20:12:11,615 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=214 (bloomFilter=true), to=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/.tmp/C/afe362d8843742188e5c63ff3c9ce7d8 2024-11-18T20:12:11,622 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/.tmp/A/12ef51908c6e4012b11635f0a3bd529b as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/A/12ef51908c6e4012b11635f0a3bd529b 2024-11-18T20:12:11,631 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/A/12ef51908c6e4012b11635f0a3bd529b, entries=200, sequenceid=214, filesize=14.2 K 2024-11-18T20:12:11,633 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/.tmp/B/1671a674abe646418de4581d5ac37ac4 as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/B/1671a674abe646418de4581d5ac37ac4 2024-11-18T20:12:11,642 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/B/1671a674abe646418de4581d5ac37ac4, entries=150, sequenceid=214, filesize=11.9 K 2024-11-18T20:12:11,643 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/.tmp/C/afe362d8843742188e5c63ff3c9ce7d8 as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/C/afe362d8843742188e5c63ff3c9ce7d8 2024-11-18T20:12:11,652 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/C/afe362d8843742188e5c63ff3c9ce7d8, entries=150, sequenceid=214, filesize=11.9 K 2024-11-18T20:12:11,654 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~281.95 KB/288720, currentSize=100.63 KB/103050 for 326884d1d40acf488e8bd50aea838c73 in 593ms, sequenceid=214, compaction requested=true 2024-11-18T20:12:11,654 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 326884d1d40acf488e8bd50aea838c73: 2024-11-18T20:12:11,654 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 326884d1d40acf488e8bd50aea838c73:A, priority=-2147483648, current under compaction store size is 1 2024-11-18T20:12:11,654 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:12:11,654 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T20:12:11,654 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 326884d1d40acf488e8bd50aea838c73:B, priority=-2147483648, current under compaction store size is 2 2024-11-18T20:12:11,654 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:12:11,654 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T20:12:11,654 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 326884d1d40acf488e8bd50aea838c73:C, priority=-2147483648, current under compaction store size is 3 2024-11-18T20:12:11,654 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T20:12:11,656 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T20:12:11,657 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HStore(1541): 326884d1d40acf488e8bd50aea838c73/B is initiating minor compaction (all files) 2024-11-18T20:12:11,657 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 326884d1d40acf488e8bd50aea838c73/B in TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73. 2024-11-18T20:12:11,657 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/B/a77f383b179545ec8b284f279bdeb156, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/B/9763ff43338141c2a7be872384367835, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/B/1671a674abe646418de4581d5ac37ac4] into tmpdir=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/.tmp, totalSize=36.0 K 2024-11-18T20:12:11,659 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.Compactor(225): Compacting a77f383b179545ec8b284f279bdeb156, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=171, earliestPutTs=1731960728481 2024-11-18T20:12:11,660 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39253 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T20:12:11,660 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HStore(1541): 326884d1d40acf488e8bd50aea838c73/A is initiating minor compaction (all files) 2024-11-18T20:12:11,660 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 326884d1d40acf488e8bd50aea838c73/A in TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73. 2024-11-18T20:12:11,660 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/A/33812c745aee4ce8bddf200a129b9f9b, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/A/62c2388048994dd2a0162d466fcc1c10, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/A/12ef51908c6e4012b11635f0a3bd529b] into tmpdir=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/.tmp, totalSize=38.3 K 2024-11-18T20:12:11,661 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.Compactor(225): Compacting 9763ff43338141c2a7be872384367835, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=193, earliestPutTs=1731960729183 2024-11-18T20:12:11,662 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] compactions.Compactor(225): Compacting 33812c745aee4ce8bddf200a129b9f9b, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=171, earliestPutTs=1731960728481 2024-11-18T20:12:11,662 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.Compactor(225): Compacting 1671a674abe646418de4581d5ac37ac4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=214, earliestPutTs=1731960729892 2024-11-18T20:12:11,662 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] compactions.Compactor(225): Compacting 62c2388048994dd2a0162d466fcc1c10, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=193, earliestPutTs=1731960729183 2024-11-18T20:12:11,663 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] compactions.Compactor(225): Compacting 12ef51908c6e4012b11635f0a3bd529b, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=214, earliestPutTs=1731960729892 2024-11-18T20:12:11,668 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35963 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-18T20:12:11,668 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73. 2024-11-18T20:12:11,668 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2902): Flushing 326884d1d40acf488e8bd50aea838c73 3/3 column families, dataSize=100.63 KB heapSize=264.42 KB 2024-11-18T20:12:11,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/.tmp/A/e0ca3c9891654ec5a4ffdb50923e34ca is 50, key is test_row_0/A:col10/1731960731085/Put/seqid=0 2024-11-18T20:12:11,705 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 326884d1d40acf488e8bd50aea838c73#B#compaction#45 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:12:11,706 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/.tmp/B/83d49644bc2840ca893579a7a3495c85 is 50, key is test_row_0/B:col10/1731960729892/Put/seqid=0 2024-11-18T20:12:11,708 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 326884d1d40acf488e8bd50aea838c73#A#compaction#46 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:12:11,709 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/.tmp/A/21ad72f14a1d41348947ef07c8c74675 is 50, key is test_row_0/A:col10/1731960729892/Put/seqid=0 2024-11-18T20:12:11,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(8855): Flush requested on 326884d1d40acf488e8bd50aea838c73 2024-11-18T20:12:11,717 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73. as already flushing 2024-11-18T20:12:11,741 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073741883_1059 (size=12663) 2024-11-18T20:12:11,753 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:11,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33056 deadline: 1731960791749, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:11,754 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:11,754 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:11,754 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:12:11,762 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073741881_1057 (size=12151) 2024-11-18T20:12:11,762 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073741882_1058 (size=12663) 2024-11-18T20:12:11,764 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:11,764 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:11,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33072 deadline: 1731960791754, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:11,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33028 deadline: 1731960791752, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:11,765 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:11,765 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:11,765 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:11,765 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:11,765 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:11,765 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:12:11,765 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:12:11,765 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:11,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33046 deadline: 1731960791754, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:11,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33070 deadline: 1731960791755, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:11,765 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:11,766 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:11,766 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:11,766 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:12:11,766 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:11,766 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:12:11,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-18T20:12:11,864 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:11,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33056 deadline: 1731960791863, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:11,865 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:11,865 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:11,865 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:12:11,874 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:11,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33028 deadline: 1731960791873, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:11,875 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:11,875 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:11,875 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:11,875 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:12:11,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33072 deadline: 1731960791873, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:11,876 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:11,876 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:11,876 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:11,876 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:12:11,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33046 deadline: 1731960791874, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:11,876 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:11,876 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:11,876 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:11,877 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:12:11,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33070 deadline: 1731960791874, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:11,877 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:11,877 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:11,877 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:12:12,076 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:12,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33056 deadline: 1731960792073, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:12,077 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:12,077 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:12,077 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:12:12,084 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:12,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33072 deadline: 1731960792083, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:12,085 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:12,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33046 deadline: 1731960792084, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:12,086 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:12,086 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:12,086 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:12:12,087 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:12,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33070 deadline: 1731960792084, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:12,088 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:12,088 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:12,088 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:12:12,088 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:12,088 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:12,088 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:12,088 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:12:12,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33028 deadline: 1731960792084, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:12,095 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:12,095 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:12,095 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:12:12,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-18T20:12:12,148 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/.tmp/A/21ad72f14a1d41348947ef07c8c74675 as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/A/21ad72f14a1d41348947ef07c8c74675 2024-11-18T20:12:12,163 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=232 (bloomFilter=true), to=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/.tmp/A/e0ca3c9891654ec5a4ffdb50923e34ca 2024-11-18T20:12:12,164 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 326884d1d40acf488e8bd50aea838c73/A of 326884d1d40acf488e8bd50aea838c73 into 21ad72f14a1d41348947ef07c8c74675(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:12:12,165 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 326884d1d40acf488e8bd50aea838c73: 2024-11-18T20:12:12,165 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., storeName=326884d1d40acf488e8bd50aea838c73/A, priority=13, startTime=1731960731654; duration=0sec 2024-11-18T20:12:12,165 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T20:12:12,165 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 326884d1d40acf488e8bd50aea838c73:A 2024-11-18T20:12:12,165 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T20:12:12,172 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/.tmp/B/83d49644bc2840ca893579a7a3495c85 as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/B/83d49644bc2840ca893579a7a3495c85 2024-11-18T20:12:12,173 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36829 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T20:12:12,173 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HStore(1541): 326884d1d40acf488e8bd50aea838c73/C is initiating minor compaction (all files) 2024-11-18T20:12:12,173 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 326884d1d40acf488e8bd50aea838c73/C in TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73. 2024-11-18T20:12:12,174 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/C/74f1d8e339d342a9ade550c080960874, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/C/427bbd64ca6947c9b52afa0d85685f9a, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/C/afe362d8843742188e5c63ff3c9ce7d8] into tmpdir=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/.tmp, totalSize=36.0 K 2024-11-18T20:12:12,174 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] compactions.Compactor(225): Compacting 74f1d8e339d342a9ade550c080960874, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=171, earliestPutTs=1731960728481 2024-11-18T20:12:12,175 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] compactions.Compactor(225): Compacting 427bbd64ca6947c9b52afa0d85685f9a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=193, earliestPutTs=1731960729183 2024-11-18T20:12:12,175 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] compactions.Compactor(225): Compacting afe362d8843742188e5c63ff3c9ce7d8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=214, earliestPutTs=1731960729892 2024-11-18T20:12:12,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/.tmp/B/535b3a85d3684883bcf403ce3cc452de is 50, key is test_row_0/B:col10/1731960731085/Put/seqid=0 2024-11-18T20:12:12,184 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 326884d1d40acf488e8bd50aea838c73/B of 326884d1d40acf488e8bd50aea838c73 into 83d49644bc2840ca893579a7a3495c85(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:12:12,184 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 326884d1d40acf488e8bd50aea838c73: 2024-11-18T20:12:12,184 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., storeName=326884d1d40acf488e8bd50aea838c73/B, priority=13, startTime=1731960731654; duration=0sec 2024-11-18T20:12:12,185 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:12:12,185 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 326884d1d40acf488e8bd50aea838c73:B 2024-11-18T20:12:12,203 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073741884_1060 (size=12151) 2024-11-18T20:12:12,211 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 326884d1d40acf488e8bd50aea838c73#C#compaction#48 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:12:12,211 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/.tmp/C/7c807d6029364cdf8693b06bc4cf0604 is 50, key is test_row_0/C:col10/1731960729892/Put/seqid=0 2024-11-18T20:12:12,238 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073741885_1061 (size=12629) 2024-11-18T20:12:12,386 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:12,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33056 deadline: 1731960792383, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:12,386 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:12,386 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:12,386 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:12:12,396 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:12,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33046 deadline: 1731960792393, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:12,397 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:12,397 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:12,397 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:12:12,397 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:12,397 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:12,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33072 deadline: 1731960792393, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:12,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33070 deadline: 1731960792393, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:12,398 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:12,398 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:12,398 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:12:12,398 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:12,398 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:12,398 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:12:12,406 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:12,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33028 deadline: 1731960792403, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:12,406 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:12,406 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:12,406 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:12:12,604 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=232 (bloomFilter=true), to=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/.tmp/B/535b3a85d3684883bcf403ce3cc452de 2024-11-18T20:12:12,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/.tmp/C/bd5835e1e6ac4244b2e746fac8937694 is 50, key is test_row_0/C:col10/1731960731085/Put/seqid=0 2024-11-18T20:12:12,627 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073741886_1062 (size=12151) 2024-11-18T20:12:12,628 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=232 (bloomFilter=true), to=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/.tmp/C/bd5835e1e6ac4244b2e746fac8937694 2024-11-18T20:12:12,635 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/.tmp/A/e0ca3c9891654ec5a4ffdb50923e34ca as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/A/e0ca3c9891654ec5a4ffdb50923e34ca 2024-11-18T20:12:12,642 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/A/e0ca3c9891654ec5a4ffdb50923e34ca, entries=150, sequenceid=232, filesize=11.9 K 2024-11-18T20:12:12,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-18T20:12:12,644 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/.tmp/B/535b3a85d3684883bcf403ce3cc452de as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/B/535b3a85d3684883bcf403ce3cc452de 2024-11-18T20:12:12,645 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/.tmp/C/7c807d6029364cdf8693b06bc4cf0604 as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/C/7c807d6029364cdf8693b06bc4cf0604 2024-11-18T20:12:12,654 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 326884d1d40acf488e8bd50aea838c73/C of 326884d1d40acf488e8bd50aea838c73 into 7c807d6029364cdf8693b06bc4cf0604(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:12:12,654 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 326884d1d40acf488e8bd50aea838c73: 2024-11-18T20:12:12,654 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., storeName=326884d1d40acf488e8bd50aea838c73/C, priority=13, startTime=1731960731654; duration=0sec 2024-11-18T20:12:12,654 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:12:12,654 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 326884d1d40acf488e8bd50aea838c73:C 2024-11-18T20:12:12,655 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/B/535b3a85d3684883bcf403ce3cc452de, entries=150, sequenceid=232, filesize=11.9 K 2024-11-18T20:12:12,657 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/.tmp/C/bd5835e1e6ac4244b2e746fac8937694 as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/C/bd5835e1e6ac4244b2e746fac8937694 2024-11-18T20:12:12,666 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/C/bd5835e1e6ac4244b2e746fac8937694, entries=150, sequenceid=232, filesize=11.9 K 2024-11-18T20:12:12,668 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~264.38 KB/270720, currentSize=107.34 KB/109920 for 326884d1d40acf488e8bd50aea838c73 in 1000ms, sequenceid=232, compaction requested=false 2024-11-18T20:12:12,668 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2603): Flush status journal for 326884d1d40acf488e8bd50aea838c73: 2024-11-18T20:12:12,668 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73. 2024-11-18T20:12:12,668 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=14 2024-11-18T20:12:12,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41277 {}] master.HMaster(4169): Remote procedure done, pid=14 2024-11-18T20:12:12,674 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13 2024-11-18T20:12:12,675 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1580 sec 2024-11-18T20:12:12,681 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees in 1.1720 sec 2024-11-18T20:12:12,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(8855): Flush requested on 326884d1d40acf488e8bd50aea838c73 2024-11-18T20:12:12,898 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-18T20:12:12,898 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 326884d1d40acf488e8bd50aea838c73 3/3 column families, dataSize=114.05 KB heapSize=299.58 KB 2024-11-18T20:12:12,906 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/.tmp/A/1bfc3b275e9f44fbb794e1e5efef9013 is 50, key is test_row_0/A:col10/1731960731751/Put/seqid=0 2024-11-18T20:12:12,922 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073741887_1063 (size=12151) 2024-11-18T20:12:12,924 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=255 (bloomFilter=true), to=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/.tmp/A/1bfc3b275e9f44fbb794e1e5efef9013 2024-11-18T20:12:12,927 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:12,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33028 deadline: 1731960792922, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:12,928 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:12,928 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:12,927 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:12,928 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:12:12,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33056 deadline: 1731960792923, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:12,928 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:12,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33072 deadline: 1731960792926, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:12,929 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:12,929 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:12,929 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:12:12,929 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:12,929 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:12,929 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:12:12,931 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:12,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33046 deadline: 1731960792928, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:12,931 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:12,931 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:12,931 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:12,931 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:12:12,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33070 deadline: 1731960792928, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:12,932 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:12,932 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:12,932 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:12:12,935 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/.tmp/B/e49e53b8d7ca4f81b7e82677ea17c404 is 50, key is test_row_0/B:col10/1731960731751/Put/seqid=0 2024-11-18T20:12:12,940 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073741888_1064 (size=12151) 2024-11-18T20:12:13,035 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:13,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33028 deadline: 1731960793033, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:13,036 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:13,036 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:13,036 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:12:13,036 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:13,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33056 deadline: 1731960793033, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:13,036 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:13,036 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:13,036 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:13,036 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:12:13,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33072 deadline: 1731960793034, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:13,036 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:13,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33046 deadline: 1731960793034, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:13,037 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:13,037 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:13,037 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:13,037 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:12:13,037 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:13,037 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:12:13,046 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:13,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33070 deadline: 1731960793043, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:13,046 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:13,046 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:13,046 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:12:13,244 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:13,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33028 deadline: 1731960793243, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:13,245 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:13,245 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:13,245 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:13,245 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:12:13,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33056 deadline: 1731960793243, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:13,245 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:13,245 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:13,245 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:12:13,245 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:13,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33072 deadline: 1731960793243, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:13,246 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:13,246 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:13,246 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:12:13,246 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:13,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33046 deadline: 1731960793243, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:13,247 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:13,247 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:13,247 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:12:13,256 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:13,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33070 deadline: 1731960793253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:13,257 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:13,257 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:13,258 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:12:13,343 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=255 (bloomFilter=true), to=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/.tmp/B/e49e53b8d7ca4f81b7e82677ea17c404 2024-11-18T20:12:13,362 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/.tmp/C/71f170f01f364dac9c12b546f53e20fa is 50, key is test_row_0/C:col10/1731960731751/Put/seqid=0 2024-11-18T20:12:13,378 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073741889_1065 (size=12151) 2024-11-18T20:12:13,380 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=255 (bloomFilter=true), to=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/.tmp/C/71f170f01f364dac9c12b546f53e20fa 2024-11-18T20:12:13,392 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/.tmp/A/1bfc3b275e9f44fbb794e1e5efef9013 as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/A/1bfc3b275e9f44fbb794e1e5efef9013 2024-11-18T20:12:13,405 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/A/1bfc3b275e9f44fbb794e1e5efef9013, entries=150, sequenceid=255, filesize=11.9 K 2024-11-18T20:12:13,406 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/.tmp/B/e49e53b8d7ca4f81b7e82677ea17c404 as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/B/e49e53b8d7ca4f81b7e82677ea17c404 2024-11-18T20:12:13,414 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/B/e49e53b8d7ca4f81b7e82677ea17c404, entries=150, sequenceid=255, filesize=11.9 K 2024-11-18T20:12:13,415 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/.tmp/C/71f170f01f364dac9c12b546f53e20fa as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/C/71f170f01f364dac9c12b546f53e20fa 2024-11-18T20:12:13,425 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/C/71f170f01f364dac9c12b546f53e20fa, entries=150, sequenceid=255, filesize=11.9 K 2024-11-18T20:12:13,428 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~299.53 KB/306720, currentSize=93.93 KB/96180 for 326884d1d40acf488e8bd50aea838c73 in 529ms, sequenceid=255, compaction requested=true 2024-11-18T20:12:13,428 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 326884d1d40acf488e8bd50aea838c73: 2024-11-18T20:12:13,428 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 326884d1d40acf488e8bd50aea838c73:A, priority=-2147483648, current under compaction store size is 1 2024-11-18T20:12:13,428 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:12:13,428 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T20:12:13,428 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 326884d1d40acf488e8bd50aea838c73:B, priority=-2147483648, current under compaction store size is 2 2024-11-18T20:12:13,428 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:12:13,428 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T20:12:13,428 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 326884d1d40acf488e8bd50aea838c73:C, priority=-2147483648, current under compaction store size is 3 2024-11-18T20:12:13,428 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T20:12:13,430 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36965 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T20:12:13,430 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HStore(1541): 326884d1d40acf488e8bd50aea838c73/A is initiating minor compaction (all files) 2024-11-18T20:12:13,431 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 326884d1d40acf488e8bd50aea838c73/A in TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73. 2024-11-18T20:12:13,431 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/A/21ad72f14a1d41348947ef07c8c74675, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/A/e0ca3c9891654ec5a4ffdb50923e34ca, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/A/1bfc3b275e9f44fbb794e1e5efef9013] into tmpdir=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/.tmp, totalSize=36.1 K 2024-11-18T20:12:13,432 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36965 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T20:12:13,432 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HStore(1541): 326884d1d40acf488e8bd50aea838c73/B is initiating minor compaction (all files) 2024-11-18T20:12:13,432 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] compactions.Compactor(225): Compacting 21ad72f14a1d41348947ef07c8c74675, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=214, earliestPutTs=1731960729892 2024-11-18T20:12:13,432 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 326884d1d40acf488e8bd50aea838c73/B in TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73. 2024-11-18T20:12:13,432 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/B/83d49644bc2840ca893579a7a3495c85, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/B/535b3a85d3684883bcf403ce3cc452de, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/B/e49e53b8d7ca4f81b7e82677ea17c404] into tmpdir=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/.tmp, totalSize=36.1 K 2024-11-18T20:12:13,432 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] compactions.Compactor(225): Compacting e0ca3c9891654ec5a4ffdb50923e34ca, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=232, earliestPutTs=1731960731081 2024-11-18T20:12:13,432 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.Compactor(225): Compacting 83d49644bc2840ca893579a7a3495c85, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=214, earliestPutTs=1731960729892 2024-11-18T20:12:13,433 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1bfc3b275e9f44fbb794e1e5efef9013, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=255, earliestPutTs=1731960731751 2024-11-18T20:12:13,433 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.Compactor(225): Compacting 535b3a85d3684883bcf403ce3cc452de, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=232, earliestPutTs=1731960731081 2024-11-18T20:12:13,433 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.Compactor(225): Compacting e49e53b8d7ca4f81b7e82677ea17c404, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=255, earliestPutTs=1731960731751 2024-11-18T20:12:13,448 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 326884d1d40acf488e8bd50aea838c73#A#compaction#53 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:12:13,448 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/.tmp/A/a4e5a680ef9248849f0b481698f0751b is 50, key is test_row_0/A:col10/1731960731751/Put/seqid=0 2024-11-18T20:12:13,453 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 326884d1d40acf488e8bd50aea838c73#B#compaction#54 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:12:13,455 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/.tmp/B/e74fd6ec3a534ac0afa00c2c9eb2e03b is 50, key is test_row_0/B:col10/1731960731751/Put/seqid=0 2024-11-18T20:12:13,479 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073741890_1066 (size=12765) 2024-11-18T20:12:13,492 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073741891_1067 (size=12765) 2024-11-18T20:12:13,505 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/.tmp/B/e74fd6ec3a534ac0afa00c2c9eb2e03b as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/B/e74fd6ec3a534ac0afa00c2c9eb2e03b 2024-11-18T20:12:13,516 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 326884d1d40acf488e8bd50aea838c73/B of 326884d1d40acf488e8bd50aea838c73 into e74fd6ec3a534ac0afa00c2c9eb2e03b(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:12:13,516 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 326884d1d40acf488e8bd50aea838c73: 2024-11-18T20:12:13,517 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., storeName=326884d1d40acf488e8bd50aea838c73/B, priority=13, startTime=1731960733428; duration=0sec 2024-11-18T20:12:13,517 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T20:12:13,517 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 326884d1d40acf488e8bd50aea838c73:B 2024-11-18T20:12:13,517 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T20:12:13,518 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36931 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T20:12:13,519 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HStore(1541): 326884d1d40acf488e8bd50aea838c73/C is initiating minor compaction (all files) 2024-11-18T20:12:13,519 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 326884d1d40acf488e8bd50aea838c73/C in TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73. 2024-11-18T20:12:13,519 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/C/7c807d6029364cdf8693b06bc4cf0604, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/C/bd5835e1e6ac4244b2e746fac8937694, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/C/71f170f01f364dac9c12b546f53e20fa] into tmpdir=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/.tmp, totalSize=36.1 K 2024-11-18T20:12:13,519 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.Compactor(225): Compacting 7c807d6029364cdf8693b06bc4cf0604, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=214, earliestPutTs=1731960729892 2024-11-18T20:12:13,520 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.Compactor(225): Compacting bd5835e1e6ac4244b2e746fac8937694, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=232, earliestPutTs=1731960731081 2024-11-18T20:12:13,521 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.Compactor(225): Compacting 71f170f01f364dac9c12b546f53e20fa, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=255, earliestPutTs=1731960731751 2024-11-18T20:12:13,534 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 326884d1d40acf488e8bd50aea838c73#C#compaction#55 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:12:13,535 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/.tmp/C/bc865bcfd2744007984ced955360e6f3 is 50, key is test_row_0/C:col10/1731960731751/Put/seqid=0 2024-11-18T20:12:13,547 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073741892_1068 (size=12731) 2024-11-18T20:12:13,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(8855): Flush requested on 326884d1d40acf488e8bd50aea838c73 2024-11-18T20:12:13,557 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-18T20:12:13,557 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 326884d1d40acf488e8bd50aea838c73 3/3 column families, dataSize=100.63 KB heapSize=264.42 KB 2024-11-18T20:12:13,582 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/.tmp/A/0c882164965a42689763478e96370b67 is 50, key is test_row_0/A:col10/1731960732925/Put/seqid=0 2024-11-18T20:12:13,586 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:13,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33072 deadline: 1731960793579, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:13,586 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:13,587 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:13,587 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:12:13,588 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:13,589 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:13,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33070 deadline: 1731960793581, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:13,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33056 deadline: 1731960793581, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:13,589 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:13,590 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:13,590 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:12:13,591 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:13,591 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:13,591 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:13,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33028 deadline: 1731960793587, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:13,591 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:12:13,591 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:13,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33046 deadline: 1731960793589, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:13,592 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:13,592 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:13,592 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:12:13,594 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:13,594 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:13,594 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:12:13,607 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073741893_1069 (size=12301) 2024-11-18T20:12:13,608 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=274 (bloomFilter=true), to=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/.tmp/A/0c882164965a42689763478e96370b67 2024-11-18T20:12:13,639 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/.tmp/B/bc34dc5c7a4440bbae1fdc8cde503796 is 50, key is test_row_0/B:col10/1731960732925/Put/seqid=0 2024-11-18T20:12:13,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-18T20:12:13,653 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-18T20:12:13,654 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41277 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-18T20:12:13,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41277 {}] procedure2.ProcedureExecutor(1139): Stored pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees 2024-11-18T20:12:13,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-18T20:12:13,660 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-18T20:12:13,661 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-18T20:12:13,661 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=16, ppid=15, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-18T20:12:13,664 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073741894_1070 (size=12301) 2024-11-18T20:12:13,665 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=274 (bloomFilter=true), to=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/.tmp/B/bc34dc5c7a4440bbae1fdc8cde503796 2024-11-18T20:12:13,680 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/.tmp/C/ff309597fdc6400bb092706f52627c2e is 50, key is test_row_0/C:col10/1731960732925/Put/seqid=0 2024-11-18T20:12:13,686 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073741895_1071 (size=12301) 2024-11-18T20:12:13,694 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:13,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33072 deadline: 1731960793693, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:13,695 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:13,695 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:13,695 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:13,695 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:12:13,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33056 deadline: 1731960793694, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:13,695 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:13,695 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:13,695 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:12:13,695 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:13,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33070 deadline: 1731960793694, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:13,696 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:13,696 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:13,696 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:12:13,705 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:13,705 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:13,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33046 deadline: 1731960793703, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:13,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33028 deadline: 1731960793703, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:13,706 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:13,706 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:13,706 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:12:13,707 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:13,707 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:13,707 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:12:13,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-18T20:12:13,815 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35963 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-18T20:12:13,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73. 2024-11-18T20:12:13,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73. as already flushing 2024-11-18T20:12:13,816 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73. 2024-11-18T20:12:13,816 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 326884d1d40acf488e8bd50aea838c73, NAME => 'TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:12:13,817 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 326884d1d40acf488e8bd50aea838c73, NAME => 'TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:12:13,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41277 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 326884d1d40acf488e8bd50aea838c73, NAME => 'TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 326884d1d40acf488e8bd50aea838c73, NAME => 'TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:12:13,887 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/.tmp/A/a4e5a680ef9248849f0b481698f0751b as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/A/a4e5a680ef9248849f0b481698f0751b 2024-11-18T20:12:13,898 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 326884d1d40acf488e8bd50aea838c73/A of 326884d1d40acf488e8bd50aea838c73 into a4e5a680ef9248849f0b481698f0751b(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:12:13,899 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 326884d1d40acf488e8bd50aea838c73: 2024-11-18T20:12:13,899 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., storeName=326884d1d40acf488e8bd50aea838c73/A, priority=13, startTime=1731960733428; duration=0sec 2024-11-18T20:12:13,901 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:12:13,901 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 326884d1d40acf488e8bd50aea838c73:A 2024-11-18T20:12:13,905 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:13,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33072 deadline: 1731960793904, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:13,906 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:13,906 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:13,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33070 deadline: 1731960793904, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:13,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33056 deadline: 1731960793904, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:13,907 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:13,907 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:13,907 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:12:13,907 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:13,907 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:13,907 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:12:13,907 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:13,907 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:13,907 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:12:13,918 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:13,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33028 deadline: 1731960793913, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:13,919 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:13,919 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:13,919 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:13,919 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:12:13,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33046 deadline: 1731960793913, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:13,920 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:13,920 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:13,920 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:12:13,960 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/.tmp/C/bc865bcfd2744007984ced955360e6f3 as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/C/bc865bcfd2744007984ced955360e6f3 2024-11-18T20:12:13,970 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 326884d1d40acf488e8bd50aea838c73/C of 326884d1d40acf488e8bd50aea838c73 into bc865bcfd2744007984ced955360e6f3(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:12:13,970 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 326884d1d40acf488e8bd50aea838c73: 2024-11-18T20:12:13,970 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., storeName=326884d1d40acf488e8bd50aea838c73/C, priority=13, startTime=1731960733428; duration=0sec 2024-11-18T20:12:13,970 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:12:13,970 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 326884d1d40acf488e8bd50aea838c73:C 2024-11-18T20:12:13,972 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35963 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-18T20:12:13,972 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73. 2024-11-18T20:12:13,972 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73. as already flushing 2024-11-18T20:12:13,972 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73. 2024-11-18T20:12:13,972 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 326884d1d40acf488e8bd50aea838c73, NAME => 'TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:12:13,972 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 326884d1d40acf488e8bd50aea838c73, NAME => 'TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:12:13,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-18T20:12:13,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41277 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 326884d1d40acf488e8bd50aea838c73, NAME => 'TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 326884d1d40acf488e8bd50aea838c73, NAME => 'TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:12:14,087 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=274 (bloomFilter=true), to=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/.tmp/C/ff309597fdc6400bb092706f52627c2e 2024-11-18T20:12:14,093 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/.tmp/A/0c882164965a42689763478e96370b67 as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/A/0c882164965a42689763478e96370b67 2024-11-18T20:12:14,099 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/A/0c882164965a42689763478e96370b67, entries=150, sequenceid=274, filesize=12.0 K 2024-11-18T20:12:14,100 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/.tmp/B/bc34dc5c7a4440bbae1fdc8cde503796 as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/B/bc34dc5c7a4440bbae1fdc8cde503796 2024-11-18T20:12:14,107 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/B/bc34dc5c7a4440bbae1fdc8cde503796, entries=150, sequenceid=274, filesize=12.0 K 2024-11-18T20:12:14,109 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/.tmp/C/ff309597fdc6400bb092706f52627c2e as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/C/ff309597fdc6400bb092706f52627c2e 2024-11-18T20:12:14,116 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/C/ff309597fdc6400bb092706f52627c2e, entries=150, sequenceid=274, filesize=12.0 K 2024-11-18T20:12:14,119 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~264.38 KB/270720, currentSize=107.34 KB/109920 for 326884d1d40acf488e8bd50aea838c73 in 561ms, sequenceid=274, compaction requested=false 2024-11-18T20:12:14,119 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 326884d1d40acf488e8bd50aea838c73: 2024-11-18T20:12:14,126 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35963 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-18T20:12:14,126 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73. 2024-11-18T20:12:14,126 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2902): Flushing 326884d1d40acf488e8bd50aea838c73 3/3 column families, dataSize=107.34 KB heapSize=282 KB 2024-11-18T20:12:14,132 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/.tmp/A/0c4edfba4c814e82ae3cdd6576d9986f is 50, key is test_row_0/A:col10/1731960733578/Put/seqid=0 2024-11-18T20:12:14,137 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073741896_1072 (size=12301) 2024-11-18T20:12:14,139 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=295 (bloomFilter=true), to=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/.tmp/A/0c4edfba4c814e82ae3cdd6576d9986f 2024-11-18T20:12:14,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/.tmp/B/7fbde023e43e44528f4bd740f951bfe4 is 50, key is test_row_0/B:col10/1731960733578/Put/seqid=0 2024-11-18T20:12:14,168 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073741897_1073 (size=12301) 2024-11-18T20:12:14,176 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=295 (bloomFilter=true), to=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/.tmp/B/7fbde023e43e44528f4bd740f951bfe4 2024-11-18T20:12:14,192 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/.tmp/C/591d46c5d1434763a6c7c077da1dce7a is 50, key is test_row_0/C:col10/1731960733578/Put/seqid=0 2024-11-18T20:12:14,203 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073741898_1074 (size=12301) 2024-11-18T20:12:14,203 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=295 (bloomFilter=true), to=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/.tmp/C/591d46c5d1434763a6c7c077da1dce7a 2024-11-18T20:12:14,212 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/.tmp/A/0c4edfba4c814e82ae3cdd6576d9986f as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/A/0c4edfba4c814e82ae3cdd6576d9986f 2024-11-18T20:12:14,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(8855): Flush requested on 326884d1d40acf488e8bd50aea838c73 2024-11-18T20:12:14,215 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73. as already flushing 2024-11-18T20:12:14,221 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/A/0c4edfba4c814e82ae3cdd6576d9986f, entries=150, sequenceid=295, filesize=12.0 K 2024-11-18T20:12:14,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/.tmp/B/7fbde023e43e44528f4bd740f951bfe4 as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/B/7fbde023e43e44528f4bd740f951bfe4 2024-11-18T20:12:14,229 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/B/7fbde023e43e44528f4bd740f951bfe4, entries=150, sequenceid=295, filesize=12.0 K 2024-11-18T20:12:14,231 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/.tmp/C/591d46c5d1434763a6c7c077da1dce7a as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/C/591d46c5d1434763a6c7c077da1dce7a 2024-11-18T20:12:14,238 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/C/591d46c5d1434763a6c7c077da1dce7a, entries=150, sequenceid=295, filesize=12.0 K 2024-11-18T20:12:14,239 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~281.95 KB/288720, currentSize=80.51 KB/82440 for 326884d1d40acf488e8bd50aea838c73 in 113ms, sequenceid=295, compaction requested=true 2024-11-18T20:12:14,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2603): Flush status journal for 326884d1d40acf488e8bd50aea838c73: 2024-11-18T20:12:14,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(8855): Flush requested on 326884d1d40acf488e8bd50aea838c73 2024-11-18T20:12:14,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73. 2024-11-18T20:12:14,239 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-18T20:12:14,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=16 2024-11-18T20:12:14,240 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 326884d1d40acf488e8bd50aea838c73 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2024-11-18T20:12:14,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41277 {}] master.HMaster(4169): Remote procedure done, pid=16 2024-11-18T20:12:14,244 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=16, resume processing ppid=15 2024-11-18T20:12:14,245 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 580 msec 2024-11-18T20:12:14,247 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/.tmp/A/eed6ca7c8b1e4f8fb698b82541d43457 is 50, key is test_row_0/A:col10/1731960734232/Put/seqid=0 2024-11-18T20:12:14,247 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees in 592 msec 2024-11-18T20:12:14,257 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073741899_1075 (size=24505) 2024-11-18T20:12:14,258 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=311 (bloomFilter=true), to=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/.tmp/A/eed6ca7c8b1e4f8fb698b82541d43457 2024-11-18T20:12:14,271 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/.tmp/B/dfeeefecd0094ceabb9c4d294273533f is 50, key is test_row_0/B:col10/1731960734232/Put/seqid=0 2024-11-18T20:12:14,275 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:14,276 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:14,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33070 deadline: 1731960794267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:14,277 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:14,277 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:14,277 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:12:14,277 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:14,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33056 deadline: 1731960794271, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:14,278 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:14,278 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:14,278 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:14,278 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:12:14,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33046 deadline: 1731960794273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:14,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33072 deadline: 1731960794267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:14,278 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:14,278 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:14,278 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:14,278 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:12:14,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33028 deadline: 1731960794275, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:14,279 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:14,279 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:14,279 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:12:14,279 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:14,279 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:14,279 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:12:14,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-18T20:12:14,283 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-18T20:12:14,284 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41277 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-18T20:12:14,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41277 {}] procedure2.ProcedureExecutor(1139): Stored pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees 2024-11-18T20:12:14,286 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073741900_1076 (size=12301) 2024-11-18T20:12:14,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-18T20:12:14,290 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-18T20:12:14,291 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-18T20:12:14,291 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=18, ppid=17, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-18T20:12:14,384 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:14,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33070 deadline: 1731960794383, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:14,384 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:14,384 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:14,384 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:14,384 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:12:14,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33028 deadline: 1731960794384, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:14,385 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:14,385 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:14,385 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:14,385 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:12:14,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33056 deadline: 1731960794384, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:14,385 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:14,385 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:14,385 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:12:14,387 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:14,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33046 deadline: 1731960794387, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:14,388 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:14,388 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:14,388 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:12:14,388 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:14,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33072 deadline: 1731960794387, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:14,389 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:14,389 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:14,389 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:12:14,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-18T20:12:14,447 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35963 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-18T20:12:14,447 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73. 2024-11-18T20:12:14,447 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73. as already flushing 2024-11-18T20:12:14,447 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73. 2024-11-18T20:12:14,447 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 326884d1d40acf488e8bd50aea838c73, NAME => 'TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:12:14,447 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 326884d1d40acf488e8bd50aea838c73, NAME => 'TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:12:14,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41277 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 326884d1d40acf488e8bd50aea838c73, NAME => 'TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 326884d1d40acf488e8bd50aea838c73, NAME => 'TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:12:14,595 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:14,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33072 deadline: 1731960794595, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:14,596 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:14,596 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:14,596 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:14,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33046 deadline: 1731960794595, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:14,596 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:12:14,596 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:14,596 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:14,596 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:12:14,597 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:14,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33028 deadline: 1731960794596, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:14,597 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:14,597 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:14,597 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:14,597 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:12:14,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33056 deadline: 1731960794597, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:14,598 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:14,598 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:14,598 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:14,598 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:12:14,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33070 deadline: 1731960794597, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:14,598 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:14,598 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:14,598 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:12:14,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-18T20:12:14,605 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35963 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-18T20:12:14,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73. 2024-11-18T20:12:14,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73. as already flushing 2024-11-18T20:12:14,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73. 2024-11-18T20:12:14,606 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 326884d1d40acf488e8bd50aea838c73, NAME => 'TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:12:14,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 326884d1d40acf488e8bd50aea838c73, NAME => 'TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:12:14,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41277 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 326884d1d40acf488e8bd50aea838c73, NAME => 'TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 326884d1d40acf488e8bd50aea838c73, NAME => 'TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:12:14,687 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=311 (bloomFilter=true), to=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/.tmp/B/dfeeefecd0094ceabb9c4d294273533f 2024-11-18T20:12:14,702 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/.tmp/C/d4e8b147be684d3099d64db920d4e498 is 50, key is test_row_0/C:col10/1731960734232/Put/seqid=0 2024-11-18T20:12:14,719 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073741901_1077 (size=12301) 2024-11-18T20:12:14,720 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=311 (bloomFilter=true), to=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/.tmp/C/d4e8b147be684d3099d64db920d4e498 2024-11-18T20:12:14,729 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/.tmp/A/eed6ca7c8b1e4f8fb698b82541d43457 as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/A/eed6ca7c8b1e4f8fb698b82541d43457 2024-11-18T20:12:14,741 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/A/eed6ca7c8b1e4f8fb698b82541d43457, entries=400, sequenceid=311, filesize=23.9 K 2024-11-18T20:12:14,743 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/.tmp/B/dfeeefecd0094ceabb9c4d294273533f as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/B/dfeeefecd0094ceabb9c4d294273533f 2024-11-18T20:12:14,751 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/B/dfeeefecd0094ceabb9c4d294273533f, entries=150, sequenceid=311, filesize=12.0 K 2024-11-18T20:12:14,752 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/.tmp/C/d4e8b147be684d3099d64db920d4e498 as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/C/d4e8b147be684d3099d64db920d4e498 2024-11-18T20:12:14,758 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/C/d4e8b147be684d3099d64db920d4e498, entries=150, sequenceid=311, filesize=12.0 K 2024-11-18T20:12:14,759 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=114.05 KB/116790 for 326884d1d40acf488e8bd50aea838c73 in 520ms, sequenceid=311, compaction requested=true 2024-11-18T20:12:14,759 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 326884d1d40acf488e8bd50aea838c73: 2024-11-18T20:12:14,760 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 326884d1d40acf488e8bd50aea838c73:A, priority=-2147483648, current under compaction store size is 1 2024-11-18T20:12:14,760 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-18T20:12:14,760 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:12:14,760 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-18T20:12:14,760 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 326884d1d40acf488e8bd50aea838c73:B, priority=-2147483648, current under compaction store size is 2 2024-11-18T20:12:14,760 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:12:14,760 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 326884d1d40acf488e8bd50aea838c73:C, priority=-2147483648, current under compaction store size is 3 2024-11-18T20:12:14,760 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T20:12:14,761 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49668 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-18T20:12:14,761 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HStore(1541): 326884d1d40acf488e8bd50aea838c73/B is initiating minor compaction (all files) 2024-11-18T20:12:14,762 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 326884d1d40acf488e8bd50aea838c73/B in TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73. 2024-11-18T20:12:14,762 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/B/e74fd6ec3a534ac0afa00c2c9eb2e03b, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/B/bc34dc5c7a4440bbae1fdc8cde503796, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/B/7fbde023e43e44528f4bd740f951bfe4, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/B/dfeeefecd0094ceabb9c4d294273533f] into tmpdir=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/.tmp, totalSize=48.5 K 2024-11-18T20:12:14,762 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 61872 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-18T20:12:14,762 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HStore(1541): 326884d1d40acf488e8bd50aea838c73/A is initiating minor compaction (all files) 2024-11-18T20:12:14,762 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 326884d1d40acf488e8bd50aea838c73/A in TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73. 2024-11-18T20:12:14,762 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/A/a4e5a680ef9248849f0b481698f0751b, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/A/0c882164965a42689763478e96370b67, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/A/0c4edfba4c814e82ae3cdd6576d9986f, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/A/eed6ca7c8b1e4f8fb698b82541d43457] into tmpdir=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/.tmp, totalSize=60.4 K 2024-11-18T20:12:14,763 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.Compactor(225): Compacting e74fd6ec3a534ac0afa00c2c9eb2e03b, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=255, earliestPutTs=1731960731751 2024-11-18T20:12:14,763 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] compactions.Compactor(225): Compacting a4e5a680ef9248849f0b481698f0751b, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=255, earliestPutTs=1731960731751 2024-11-18T20:12:14,763 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0c882164965a42689763478e96370b67, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=274, earliestPutTs=1731960732925 2024-11-18T20:12:14,763 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.Compactor(225): Compacting bc34dc5c7a4440bbae1fdc8cde503796, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=274, earliestPutTs=1731960732925 2024-11-18T20:12:14,763 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0c4edfba4c814e82ae3cdd6576d9986f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=295, earliestPutTs=1731960733578 2024-11-18T20:12:14,764 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.Compactor(225): Compacting 7fbde023e43e44528f4bd740f951bfe4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=295, earliestPutTs=1731960733578 2024-11-18T20:12:14,764 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] compactions.Compactor(225): Compacting eed6ca7c8b1e4f8fb698b82541d43457, keycount=400, bloomtype=ROW, size=23.9 K, encoding=NONE, compression=NONE, seqNum=311, earliestPutTs=1731960734222 2024-11-18T20:12:14,764 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.Compactor(225): Compacting dfeeefecd0094ceabb9c4d294273533f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=311, earliestPutTs=1731960734229 2024-11-18T20:12:14,766 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35963 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-18T20:12:14,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73. 2024-11-18T20:12:14,767 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2902): Flushing 326884d1d40acf488e8bd50aea838c73 3/3 column families, dataSize=114.05 KB heapSize=299.58 KB 2024-11-18T20:12:14,774 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/.tmp/A/7e650756d1ee489cb67cc1801bee88da is 50, key is test_row_0/A:col10/1731960734270/Put/seqid=0 2024-11-18T20:12:14,788 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 326884d1d40acf488e8bd50aea838c73#B#compaction#66 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:12:14,788 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 326884d1d40acf488e8bd50aea838c73#A#compaction#67 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:12:14,788 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/.tmp/B/d602ca52155d4bbba2fa42148d2181b2 is 50, key is test_row_0/B:col10/1731960734232/Put/seqid=0 2024-11-18T20:12:14,790 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/.tmp/A/54c56f57a25247189b02168d4a6babd4 is 50, key is test_row_0/A:col10/1731960734232/Put/seqid=0 2024-11-18T20:12:14,800 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073741902_1078 (size=12301) 2024-11-18T20:12:14,800 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=331 (bloomFilter=true), to=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/.tmp/A/7e650756d1ee489cb67cc1801bee88da 2024-11-18T20:12:14,819 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073741903_1079 (size=13051) 2024-11-18T20:12:14,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/.tmp/B/d5a9d670ca964a7d88a1c8cd11d2df8e is 50, key is test_row_0/B:col10/1731960734270/Put/seqid=0 2024-11-18T20:12:14,827 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073741904_1080 (size=13051) 2024-11-18T20:12:14,842 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/.tmp/A/54c56f57a25247189b02168d4a6babd4 as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/A/54c56f57a25247189b02168d4a6babd4 2024-11-18T20:12:14,847 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073741905_1081 (size=12301) 2024-11-18T20:12:14,848 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=331 (bloomFilter=true), to=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/.tmp/B/d5a9d670ca964a7d88a1c8cd11d2df8e 2024-11-18T20:12:14,853 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 326884d1d40acf488e8bd50aea838c73/A of 326884d1d40acf488e8bd50aea838c73 into 54c56f57a25247189b02168d4a6babd4(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:12:14,854 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 326884d1d40acf488e8bd50aea838c73: 2024-11-18T20:12:14,854 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., storeName=326884d1d40acf488e8bd50aea838c73/A, priority=12, startTime=1731960734759; duration=0sec 2024-11-18T20:12:14,854 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T20:12:14,854 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 326884d1d40acf488e8bd50aea838c73:A 2024-11-18T20:12:14,855 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-18T20:12:14,858 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49634 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-18T20:12:14,858 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HStore(1541): 326884d1d40acf488e8bd50aea838c73/C is initiating minor compaction (all files) 2024-11-18T20:12:14,859 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 326884d1d40acf488e8bd50aea838c73/C in TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73. 2024-11-18T20:12:14,859 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/C/bc865bcfd2744007984ced955360e6f3, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/C/ff309597fdc6400bb092706f52627c2e, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/C/591d46c5d1434763a6c7c077da1dce7a, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/C/d4e8b147be684d3099d64db920d4e498] into tmpdir=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/.tmp, totalSize=48.5 K 2024-11-18T20:12:14,861 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] compactions.Compactor(225): Compacting bc865bcfd2744007984ced955360e6f3, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=255, earliestPutTs=1731960731751 2024-11-18T20:12:14,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/.tmp/C/0fcb7ac7831f4238a2d626df07f5cf08 is 50, key is test_row_0/C:col10/1731960734270/Put/seqid=0 2024-11-18T20:12:14,863 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] compactions.Compactor(225): Compacting ff309597fdc6400bb092706f52627c2e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=274, earliestPutTs=1731960732925 2024-11-18T20:12:14,864 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] compactions.Compactor(225): Compacting 591d46c5d1434763a6c7c077da1dce7a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=295, earliestPutTs=1731960733578 2024-11-18T20:12:14,865 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] compactions.Compactor(225): Compacting d4e8b147be684d3099d64db920d4e498, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=311, earliestPutTs=1731960734229 2024-11-18T20:12:14,877 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073741906_1082 (size=12301) 2024-11-18T20:12:14,884 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=331 (bloomFilter=true), to=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/.tmp/C/0fcb7ac7831f4238a2d626df07f5cf08 2024-11-18T20:12:14,897 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/.tmp/A/7e650756d1ee489cb67cc1801bee88da as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/A/7e650756d1ee489cb67cc1801bee88da 2024-11-18T20:12:14,908 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 326884d1d40acf488e8bd50aea838c73#C#compaction#70 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:12:14,908 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/.tmp/C/56d32afce2e246c6baac00f288e84104 is 50, key is test_row_0/C:col10/1731960734232/Put/seqid=0 2024-11-18T20:12:14,909 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/A/7e650756d1ee489cb67cc1801bee88da, entries=150, sequenceid=331, filesize=12.0 K 2024-11-18T20:12:14,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(8855): Flush requested on 326884d1d40acf488e8bd50aea838c73 2024-11-18T20:12:14,910 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/.tmp/B/d5a9d670ca964a7d88a1c8cd11d2df8e as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/B/d5a9d670ca964a7d88a1c8cd11d2df8e 2024-11-18T20:12:14,910 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73. as already flushing 2024-11-18T20:12:14,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-18T20:12:14,921 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/B/d5a9d670ca964a7d88a1c8cd11d2df8e, entries=150, sequenceid=331, filesize=12.0 K 2024-11-18T20:12:14,923 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/.tmp/C/0fcb7ac7831f4238a2d626df07f5cf08 as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/C/0fcb7ac7831f4238a2d626df07f5cf08 2024-11-18T20:12:14,926 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073741907_1083 (size=13017) 2024-11-18T20:12:14,929 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/C/0fcb7ac7831f4238a2d626df07f5cf08, entries=150, sequenceid=331, filesize=12.0 K 2024-11-18T20:12:14,931 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~299.53 KB/306720, currentSize=60.38 KB/61830 for 326884d1d40acf488e8bd50aea838c73 in 164ms, sequenceid=331, compaction requested=false 2024-11-18T20:12:14,931 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2603): Flush status journal for 326884d1d40acf488e8bd50aea838c73: 2024-11-18T20:12:14,931 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73. 2024-11-18T20:12:14,931 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=18 2024-11-18T20:12:14,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41277 {}] master.HMaster(4169): Remote procedure done, pid=18 2024-11-18T20:12:14,936 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/.tmp/C/56d32afce2e246c6baac00f288e84104 as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/C/56d32afce2e246c6baac00f288e84104 2024-11-18T20:12:14,936 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-18T20:12:14,936 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 326884d1d40acf488e8bd50aea838c73 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-18T20:12:14,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(8855): Flush requested on 326884d1d40acf488e8bd50aea838c73 2024-11-18T20:12:14,938 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=18, resume processing ppid=17 2024-11-18T20:12:14,938 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 642 msec 2024-11-18T20:12:14,944 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees in 656 msec 2024-11-18T20:12:14,948 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/.tmp/A/036d06f8299e421ca2d59e975ba5b121 is 50, key is test_row_0/A:col10/1731960734936/Put/seqid=0 2024-11-18T20:12:14,949 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 326884d1d40acf488e8bd50aea838c73/C of 326884d1d40acf488e8bd50aea838c73 into 56d32afce2e246c6baac00f288e84104(size=12.7 K), total size for store is 24.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:12:14,949 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 326884d1d40acf488e8bd50aea838c73: 2024-11-18T20:12:14,949 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., storeName=326884d1d40acf488e8bd50aea838c73/C, priority=12, startTime=1731960734760; duration=0sec 2024-11-18T20:12:14,949 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:12:14,949 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 326884d1d40acf488e8bd50aea838c73:C 2024-11-18T20:12:14,970 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073741908_1084 (size=17181) 2024-11-18T20:12:14,996 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:14,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33056 deadline: 1731960794986, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:14,997 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:14,997 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:14,997 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:14,997 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:12:14,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33072 deadline: 1731960794987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:14,998 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:14,998 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:14,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33028 deadline: 1731960794988, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:14,998 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:14,998 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:14,998 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:12:14,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33070 deadline: 1731960794989, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:14,998 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:14,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33046 deadline: 1731960794993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:14,999 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:14,999 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:14,999 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:12:14,999 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:14,999 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:14,999 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:12:14,999 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:15,000 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:15,000 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:12:15,107 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:15,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33056 deadline: 1731960795103, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:15,108 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:15,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33028 deadline: 1731960795103, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:15,108 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:15,108 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:15,108 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:12:15,109 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:15,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33070 deadline: 1731960795104, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:15,109 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:15,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33072 deadline: 1731960795104, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:15,110 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:15,110 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:15,110 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:15,110 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:12:15,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33046 deadline: 1731960795104, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:15,110 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:15,111 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:15,111 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:12:15,113 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:15,113 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:15,113 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:12:15,114 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:15,114 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:15,114 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:12:15,235 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/.tmp/B/d602ca52155d4bbba2fa42148d2181b2 as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/B/d602ca52155d4bbba2fa42148d2181b2 2024-11-18T20:12:15,245 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 326884d1d40acf488e8bd50aea838c73/B of 326884d1d40acf488e8bd50aea838c73 into d602ca52155d4bbba2fa42148d2181b2(size=12.7 K), total size for store is 24.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:12:15,245 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 326884d1d40acf488e8bd50aea838c73: 2024-11-18T20:12:15,245 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., storeName=326884d1d40acf488e8bd50aea838c73/B, priority=12, startTime=1731960734760; duration=0sec 2024-11-18T20:12:15,246 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:12:15,246 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 326884d1d40acf488e8bd50aea838c73:B 2024-11-18T20:12:15,316 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:15,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33072 deadline: 1731960795313, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:15,316 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:15,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33046 deadline: 1731960795314, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:15,317 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:15,317 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:15,317 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:12:15,315 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:15,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33056 deadline: 1731960795313, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:15,319 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:15,319 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:15,319 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:12:15,319 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:15,319 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:15,319 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:12:15,330 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:15,331 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:15,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33028 deadline: 1731960795323, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:15,331 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:15,331 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:15,331 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:12:15,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33070 deadline: 1731960795323, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:15,333 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:15,333 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:15,333 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:12:15,372 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=346 (bloomFilter=true), to=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/.tmp/A/036d06f8299e421ca2d59e975ba5b121 2024-11-18T20:12:15,382 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/.tmp/B/3db78eb034354c9abe17c0901ce2e61a is 50, key is test_row_0/B:col10/1731960734936/Put/seqid=0 2024-11-18T20:12:15,397 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073741909_1085 (size=12301) 2024-11-18T20:12:15,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-18T20:12:15,423 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-18T20:12:15,424 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41277 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-18T20:12:15,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41277 {}] procedure2.ProcedureExecutor(1139): Stored pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees 2024-11-18T20:12:15,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-18T20:12:15,426 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-18T20:12:15,427 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-18T20:12:15,427 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=20, ppid=19, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-18T20:12:15,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-18T20:12:15,579 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35963 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-18T20:12:15,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73. 2024-11-18T20:12:15,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73. as already flushing 2024-11-18T20:12:15,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73. 2024-11-18T20:12:15,580 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 326884d1d40acf488e8bd50aea838c73, NAME => 'TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:12:15,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 326884d1d40acf488e8bd50aea838c73, NAME => 'TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:12:15,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41277 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 326884d1d40acf488e8bd50aea838c73, NAME => 'TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 326884d1d40acf488e8bd50aea838c73, NAME => 'TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:12:15,625 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:15,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33072 deadline: 1731960795623, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:15,626 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:15,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33056 deadline: 1731960795623, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:15,627 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:15,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33046 deadline: 1731960795623, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:15,627 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:15,627 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:15,627 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:12:15,628 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:15,628 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:15,628 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:12:15,629 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:15,629 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:15,629 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:12:15,643 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:15,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33028 deadline: 1731960795643, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:15,644 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:15,644 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:15,644 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:12:15,645 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:15,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33070 deadline: 1731960795644, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:15,647 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:15,647 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:15,647 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:12:15,733 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35963 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-18T20:12:15,733 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73. 2024-11-18T20:12:15,733 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73. as already flushing 2024-11-18T20:12:15,733 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73. 2024-11-18T20:12:15,733 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 326884d1d40acf488e8bd50aea838c73, NAME => 'TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:12:15,734 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 326884d1d40acf488e8bd50aea838c73, NAME => 'TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:12:15,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41277 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 326884d1d40acf488e8bd50aea838c73, NAME => 'TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 326884d1d40acf488e8bd50aea838c73, NAME => 'TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:12:15,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-18T20:12:15,797 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=346 (bloomFilter=true), to=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/.tmp/B/3db78eb034354c9abe17c0901ce2e61a 2024-11-18T20:12:15,815 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/.tmp/C/be76e94061504f36b664767f51e08c86 is 50, key is test_row_0/C:col10/1731960734936/Put/seqid=0 2024-11-18T20:12:15,822 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073741910_1086 (size=12301) 2024-11-18T20:12:15,887 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35963 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-18T20:12:15,887 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73. 2024-11-18T20:12:15,887 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73. as already flushing 2024-11-18T20:12:15,887 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73. 2024-11-18T20:12:15,888 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 326884d1d40acf488e8bd50aea838c73, NAME => 'TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:12:15,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 326884d1d40acf488e8bd50aea838c73, NAME => 'TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:12:15,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41277 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 326884d1d40acf488e8bd50aea838c73, NAME => 'TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 326884d1d40acf488e8bd50aea838c73, NAME => 'TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:12:16,044 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35963 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-18T20:12:16,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73. 2024-11-18T20:12:16,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73. as already flushing 2024-11-18T20:12:16,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73. 2024-11-18T20:12:16,045 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 326884d1d40acf488e8bd50aea838c73, NAME => 'TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:12:16,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 326884d1d40acf488e8bd50aea838c73, NAME => 'TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:12:16,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41277 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 326884d1d40acf488e8bd50aea838c73, NAME => 'TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 326884d1d40acf488e8bd50aea838c73, NAME => 'TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:12:16,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-18T20:12:16,135 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:16,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33072 deadline: 1731960796133, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:16,136 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:16,136 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:16,136 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:16,136 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:12:16,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33056 deadline: 1731960796133, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:16,136 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:16,136 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:16,136 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:12:16,136 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:16,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33046 deadline: 1731960796133, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:16,137 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:16,137 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:16,137 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:12:16,154 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:16,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33028 deadline: 1731960796153, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:16,155 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:16,155 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:16,155 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:16,155 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:12:16,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33070 deadline: 1731960796153, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:16,155 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:16,155 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:16,155 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:12:16,199 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35963 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-18T20:12:16,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73. 2024-11-18T20:12:16,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73. as already flushing 2024-11-18T20:12:16,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73. 2024-11-18T20:12:16,200 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 326884d1d40acf488e8bd50aea838c73, NAME => 'TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:12:16,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 326884d1d40acf488e8bd50aea838c73, NAME => 'TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:12:16,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41277 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 326884d1d40acf488e8bd50aea838c73, NAME => 'TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 326884d1d40acf488e8bd50aea838c73, NAME => 'TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:12:16,223 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=346 (bloomFilter=true), to=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/.tmp/C/be76e94061504f36b664767f51e08c86 2024-11-18T20:12:16,228 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/.tmp/A/036d06f8299e421ca2d59e975ba5b121 as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/A/036d06f8299e421ca2d59e975ba5b121 2024-11-18T20:12:16,232 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/A/036d06f8299e421ca2d59e975ba5b121, entries=250, sequenceid=346, filesize=16.8 K 2024-11-18T20:12:16,233 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/.tmp/B/3db78eb034354c9abe17c0901ce2e61a as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/B/3db78eb034354c9abe17c0901ce2e61a 2024-11-18T20:12:16,240 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/B/3db78eb034354c9abe17c0901ce2e61a, entries=150, sequenceid=346, filesize=12.0 K 2024-11-18T20:12:16,241 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/.tmp/C/be76e94061504f36b664767f51e08c86 as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/C/be76e94061504f36b664767f51e08c86 2024-11-18T20:12:16,248 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/C/be76e94061504f36b664767f51e08c86, entries=150, sequenceid=346, filesize=12.0 K 2024-11-18T20:12:16,249 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for 326884d1d40acf488e8bd50aea838c73 in 1313ms, sequenceid=346, compaction requested=true 2024-11-18T20:12:16,250 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 326884d1d40acf488e8bd50aea838c73: 2024-11-18T20:12:16,250 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 326884d1d40acf488e8bd50aea838c73:A, priority=-2147483648, current under compaction store size is 1 2024-11-18T20:12:16,250 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:12:16,250 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T20:12:16,250 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 326884d1d40acf488e8bd50aea838c73:B, priority=-2147483648, current under compaction store size is 2 2024-11-18T20:12:16,250 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:12:16,250 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T20:12:16,250 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 326884d1d40acf488e8bd50aea838c73:C, priority=-2147483648, current under compaction store size is 3 2024-11-18T20:12:16,250 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T20:12:16,251 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T20:12:16,251 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 42533 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T20:12:16,251 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HStore(1541): 326884d1d40acf488e8bd50aea838c73/A is initiating minor compaction (all files) 2024-11-18T20:12:16,251 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HStore(1541): 326884d1d40acf488e8bd50aea838c73/B is initiating minor compaction (all files) 2024-11-18T20:12:16,252 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 326884d1d40acf488e8bd50aea838c73/B in TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73. 2024-11-18T20:12:16,252 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 326884d1d40acf488e8bd50aea838c73/A in TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73. 2024-11-18T20:12:16,252 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/B/d602ca52155d4bbba2fa42148d2181b2, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/B/d5a9d670ca964a7d88a1c8cd11d2df8e, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/B/3db78eb034354c9abe17c0901ce2e61a] into tmpdir=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/.tmp, totalSize=36.8 K 2024-11-18T20:12:16,252 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/A/54c56f57a25247189b02168d4a6babd4, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/A/7e650756d1ee489cb67cc1801bee88da, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/A/036d06f8299e421ca2d59e975ba5b121] into tmpdir=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/.tmp, totalSize=41.5 K 2024-11-18T20:12:16,252 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] compactions.Compactor(225): Compacting 54c56f57a25247189b02168d4a6babd4, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=311, earliestPutTs=1731960734229 2024-11-18T20:12:16,252 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.Compactor(225): Compacting d602ca52155d4bbba2fa42148d2181b2, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=311, earliestPutTs=1731960734229 2024-11-18T20:12:16,253 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7e650756d1ee489cb67cc1801bee88da, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=331, earliestPutTs=1731960734270 2024-11-18T20:12:16,253 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.Compactor(225): Compacting d5a9d670ca964a7d88a1c8cd11d2df8e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=331, earliestPutTs=1731960734270 2024-11-18T20:12:16,253 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] compactions.Compactor(225): Compacting 036d06f8299e421ca2d59e975ba5b121, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=346, earliestPutTs=1731960734917 2024-11-18T20:12:16,253 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.Compactor(225): Compacting 3db78eb034354c9abe17c0901ce2e61a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=346, earliestPutTs=1731960734920 2024-11-18T20:12:16,263 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 326884d1d40acf488e8bd50aea838c73#A#compaction#74 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:12:16,263 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 326884d1d40acf488e8bd50aea838c73#B#compaction#75 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:12:16,263 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/.tmp/A/8cb3994e0f2c45538a70a12c7f7d5007 is 50, key is test_row_0/A:col10/1731960734936/Put/seqid=0 2024-11-18T20:12:16,263 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/.tmp/B/35279005022e483f8c8b7d07426de443 is 50, key is test_row_0/B:col10/1731960734936/Put/seqid=0 2024-11-18T20:12:16,279 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073741912_1088 (size=13153) 2024-11-18T20:12:16,280 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073741911_1087 (size=13153) 2024-11-18T20:12:16,353 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35963 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-18T20:12:16,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73. 2024-11-18T20:12:16,353 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2902): Flushing 326884d1d40acf488e8bd50aea838c73 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-18T20:12:16,367 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/.tmp/A/8d505235538c4e00a2e80fc5deba2fb8 is 50, key is test_row_0/A:col10/1731960734970/Put/seqid=0 2024-11-18T20:12:16,383 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073741913_1089 (size=12301) 2024-11-18T20:12:16,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-18T20:12:16,691 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/.tmp/A/8cb3994e0f2c45538a70a12c7f7d5007 as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/A/8cb3994e0f2c45538a70a12c7f7d5007 2024-11-18T20:12:16,693 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/.tmp/B/35279005022e483f8c8b7d07426de443 as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/B/35279005022e483f8c8b7d07426de443 2024-11-18T20:12:16,706 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 326884d1d40acf488e8bd50aea838c73/B of 326884d1d40acf488e8bd50aea838c73 into 35279005022e483f8c8b7d07426de443(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:12:16,706 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 326884d1d40acf488e8bd50aea838c73: 2024-11-18T20:12:16,706 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., storeName=326884d1d40acf488e8bd50aea838c73/B, priority=13, startTime=1731960736250; duration=0sec 2024-11-18T20:12:16,706 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T20:12:16,706 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 326884d1d40acf488e8bd50aea838c73:B 2024-11-18T20:12:16,706 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T20:12:16,707 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 326884d1d40acf488e8bd50aea838c73/A of 326884d1d40acf488e8bd50aea838c73 into 8cb3994e0f2c45538a70a12c7f7d5007(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:12:16,709 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 326884d1d40acf488e8bd50aea838c73: 2024-11-18T20:12:16,709 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., storeName=326884d1d40acf488e8bd50aea838c73/A, priority=13, startTime=1731960736250; duration=0sec 2024-11-18T20:12:16,711 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T20:12:16,711 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HStore(1541): 326884d1d40acf488e8bd50aea838c73/C is initiating minor compaction (all files) 2024-11-18T20:12:16,711 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 326884d1d40acf488e8bd50aea838c73/C in TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73. 2024-11-18T20:12:16,711 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/C/56d32afce2e246c6baac00f288e84104, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/C/0fcb7ac7831f4238a2d626df07f5cf08, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/C/be76e94061504f36b664767f51e08c86] into tmpdir=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/.tmp, totalSize=36.7 K 2024-11-18T20:12:16,712 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.Compactor(225): Compacting 56d32afce2e246c6baac00f288e84104, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=311, earliestPutTs=1731960734229 2024-11-18T20:12:16,712 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.Compactor(225): Compacting 0fcb7ac7831f4238a2d626df07f5cf08, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=331, earliestPutTs=1731960734270 2024-11-18T20:12:16,713 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.Compactor(225): Compacting be76e94061504f36b664767f51e08c86, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=346, earliestPutTs=1731960734920 2024-11-18T20:12:16,714 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:12:16,714 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 326884d1d40acf488e8bd50aea838c73:A 2024-11-18T20:12:16,723 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 326884d1d40acf488e8bd50aea838c73#C#compaction#77 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:12:16,724 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/.tmp/C/f472bbab37374b50b653dca80bbc51ac is 50, key is test_row_0/C:col10/1731960734936/Put/seqid=0 2024-11-18T20:12:16,732 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073741914_1090 (size=13119) 2024-11-18T20:12:16,740 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/.tmp/C/f472bbab37374b50b653dca80bbc51ac as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/C/f472bbab37374b50b653dca80bbc51ac 2024-11-18T20:12:16,749 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 326884d1d40acf488e8bd50aea838c73/C of 326884d1d40acf488e8bd50aea838c73 into f472bbab37374b50b653dca80bbc51ac(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:12:16,749 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 326884d1d40acf488e8bd50aea838c73: 2024-11-18T20:12:16,749 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., storeName=326884d1d40acf488e8bd50aea838c73/C, priority=13, startTime=1731960736250; duration=0sec 2024-11-18T20:12:16,749 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:12:16,749 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 326884d1d40acf488e8bd50aea838c73:C 2024-11-18T20:12:16,785 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=370 (bloomFilter=true), to=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/.tmp/A/8d505235538c4e00a2e80fc5deba2fb8 2024-11-18T20:12:16,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/.tmp/B/660b3de1c9994802b0f9769e933b7fa0 is 50, key is test_row_0/B:col10/1731960734970/Put/seqid=0 2024-11-18T20:12:16,843 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073741915_1091 (size=12301) 2024-11-18T20:12:16,846 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=370 (bloomFilter=true), to=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/.tmp/B/660b3de1c9994802b0f9769e933b7fa0 2024-11-18T20:12:16,857 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/.tmp/C/1a4e8da4738b4b9bb9fc00e6ff0346f4 is 50, key is test_row_0/C:col10/1731960734970/Put/seqid=0 2024-11-18T20:12:16,876 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073741916_1092 (size=12301) 2024-11-18T20:12:16,879 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=370 (bloomFilter=true), to=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/.tmp/C/1a4e8da4738b4b9bb9fc00e6ff0346f4 2024-11-18T20:12:16,887 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/.tmp/A/8d505235538c4e00a2e80fc5deba2fb8 as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/A/8d505235538c4e00a2e80fc5deba2fb8 2024-11-18T20:12:16,897 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/A/8d505235538c4e00a2e80fc5deba2fb8, entries=150, sequenceid=370, filesize=12.0 K 2024-11-18T20:12:16,898 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/.tmp/B/660b3de1c9994802b0f9769e933b7fa0 as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/B/660b3de1c9994802b0f9769e933b7fa0 2024-11-18T20:12:16,909 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/B/660b3de1c9994802b0f9769e933b7fa0, entries=150, sequenceid=370, filesize=12.0 K 2024-11-18T20:12:16,911 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/.tmp/C/1a4e8da4738b4b9bb9fc00e6ff0346f4 as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/C/1a4e8da4738b4b9bb9fc00e6ff0346f4 2024-11-18T20:12:16,921 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/C/1a4e8da4738b4b9bb9fc00e6ff0346f4, entries=150, sequenceid=370, filesize=12.0 K 2024-11-18T20:12:16,922 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=0 B/0 for 326884d1d40acf488e8bd50aea838c73 in 569ms, sequenceid=370, compaction requested=false 2024-11-18T20:12:16,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2603): Flush status journal for 326884d1d40acf488e8bd50aea838c73: 2024-11-18T20:12:16,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73. 2024-11-18T20:12:16,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=20 2024-11-18T20:12:16,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41277 {}] master.HMaster(4169): Remote procedure done, pid=20 2024-11-18T20:12:16,928 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=20, resume processing ppid=19 2024-11-18T20:12:16,928 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, ppid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4970 sec 2024-11-18T20:12:16,932 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees in 1.5050 sec 2024-11-18T20:12:17,174 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-18T20:12:17,174 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 326884d1d40acf488e8bd50aea838c73 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-18T20:12:17,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(8855): Flush requested on 326884d1d40acf488e8bd50aea838c73 2024-11-18T20:12:17,191 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/.tmp/A/4adf08c5d45f42d895d681c86b015852 is 50, key is test_row_0/A:col10/1731960737170/Put/seqid=0 2024-11-18T20:12:17,199 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073741917_1093 (size=17181) 2024-11-18T20:12:17,207 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=385 (bloomFilter=true), to=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/.tmp/A/4adf08c5d45f42d895d681c86b015852 2024-11-18T20:12:17,225 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/.tmp/B/be10d891706146c6a4715cb1cac38f71 is 50, key is test_row_0/B:col10/1731960737170/Put/seqid=0 2024-11-18T20:12:17,237 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:17,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33070 deadline: 1731960797231, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:17,238 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:17,238 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:17,238 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:12:17,242 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:17,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33028 deadline: 1731960797233, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:17,243 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:17,243 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:17,243 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:12:17,247 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:17,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33072 deadline: 1731960797236, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:17,248 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:17,248 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:17,248 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:12:17,248 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:17,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33046 deadline: 1731960797237, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:17,252 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:17,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33056 deadline: 1731960797238, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:17,259 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:17,259 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:17,259 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:12:17,260 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:17,260 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:17,260 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:12:17,271 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073741918_1094 (size=12301) 2024-11-18T20:12:17,348 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:17,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33070 deadline: 1731960797346, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:17,349 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:17,349 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:17,349 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:12:17,356 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:17,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33028 deadline: 1731960797353, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:17,357 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:17,357 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:17,357 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:12:17,357 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:17,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33072 deadline: 1731960797353, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:17,358 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:17,358 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:17,358 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:12:17,369 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:17,369 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:17,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33056 deadline: 1731960797364, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:17,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33046 deadline: 1731960797364, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:17,370 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:17,370 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:17,370 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:12:17,370 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:17,370 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:17,370 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:12:17,557 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:17,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33070 deadline: 1731960797553, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:17,558 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:17,558 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:17,558 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:12:17,563 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:17,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33028 deadline: 1731960797563, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:17,564 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:17,564 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:17,564 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:12:17,565 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:17,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33072 deadline: 1731960797563, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:17,565 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:17,566 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:17,566 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:12:17,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-18T20:12:17,574 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-18T20:12:17,575 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41277 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-18T20:12:17,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41277 {}] procedure2.ProcedureExecutor(1139): Stored pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees 2024-11-18T20:12:17,577 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:17,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33056 deadline: 1731960797574, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:17,577 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:17,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33046 deadline: 1731960797574, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:17,578 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:17,578 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:17,578 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:17,578 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:17,578 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:12:17,578 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:12:17,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-18T20:12:17,579 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-18T20:12:17,580 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-18T20:12:17,580 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=22, ppid=21, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-18T20:12:17,673 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=385 (bloomFilter=true), to=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/.tmp/B/be10d891706146c6a4715cb1cac38f71 2024-11-18T20:12:17,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-18T20:12:17,694 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/.tmp/C/d318e0d36ef04bbfa81b4a9cec0bed57 is 50, key is test_row_0/C:col10/1731960737170/Put/seqid=0 2024-11-18T20:12:17,722 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073741919_1095 (size=12301) 2024-11-18T20:12:17,722 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=385 (bloomFilter=true), to=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/.tmp/C/d318e0d36ef04bbfa81b4a9cec0bed57 2024-11-18T20:12:17,736 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/.tmp/A/4adf08c5d45f42d895d681c86b015852 as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/A/4adf08c5d45f42d895d681c86b015852 2024-11-18T20:12:17,737 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35963 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-18T20:12:17,737 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73. 2024-11-18T20:12:17,737 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73. as already flushing 2024-11-18T20:12:17,738 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73. 2024-11-18T20:12:17,738 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 326884d1d40acf488e8bd50aea838c73, NAME => 'TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:12:17,738 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 326884d1d40acf488e8bd50aea838c73, NAME => 'TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:12:17,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41277 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 326884d1d40acf488e8bd50aea838c73, NAME => 'TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 326884d1d40acf488e8bd50aea838c73, NAME => 'TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:12:17,764 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/A/4adf08c5d45f42d895d681c86b015852, entries=250, sequenceid=385, filesize=16.8 K 2024-11-18T20:12:17,765 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/.tmp/B/be10d891706146c6a4715cb1cac38f71 as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/B/be10d891706146c6a4715cb1cac38f71 2024-11-18T20:12:17,772 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/B/be10d891706146c6a4715cb1cac38f71, entries=150, sequenceid=385, filesize=12.0 K 2024-11-18T20:12:17,774 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/.tmp/C/d318e0d36ef04bbfa81b4a9cec0bed57 as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/C/d318e0d36ef04bbfa81b4a9cec0bed57 2024-11-18T20:12:17,784 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/C/d318e0d36ef04bbfa81b4a9cec0bed57, entries=150, sequenceid=385, filesize=12.0 K 2024-11-18T20:12:17,785 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=147.60 KB/151140 for 326884d1d40acf488e8bd50aea838c73 in 611ms, sequenceid=385, compaction requested=true 2024-11-18T20:12:17,786 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 326884d1d40acf488e8bd50aea838c73: 2024-11-18T20:12:17,786 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 326884d1d40acf488e8bd50aea838c73:A, priority=-2147483648, current under compaction store size is 1 2024-11-18T20:12:17,786 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T20:12:17,788 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:12:17,788 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T20:12:17,788 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 42635 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T20:12:17,788 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HStore(1541): 326884d1d40acf488e8bd50aea838c73/A is initiating minor compaction (all files) 2024-11-18T20:12:17,788 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 326884d1d40acf488e8bd50aea838c73/A in TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73. 2024-11-18T20:12:17,789 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/A/8cb3994e0f2c45538a70a12c7f7d5007, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/A/8d505235538c4e00a2e80fc5deba2fb8, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/A/4adf08c5d45f42d895d681c86b015852] into tmpdir=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/.tmp, totalSize=41.6 K 2024-11-18T20:12:17,789 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 326884d1d40acf488e8bd50aea838c73:B, priority=-2147483648, current under compaction store size is 2 2024-11-18T20:12:17,789 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8cb3994e0f2c45538a70a12c7f7d5007, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=346, earliestPutTs=1731960734920 2024-11-18T20:12:17,789 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:12:17,790 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 326884d1d40acf488e8bd50aea838c73:C, priority=-2147483648, current under compaction store size is 3 2024-11-18T20:12:17,790 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37755 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T20:12:17,790 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T20:12:17,790 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HStore(1541): 326884d1d40acf488e8bd50aea838c73/B is initiating minor compaction (all files) 2024-11-18T20:12:17,790 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 326884d1d40acf488e8bd50aea838c73/B in TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73. 2024-11-18T20:12:17,791 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/B/35279005022e483f8c8b7d07426de443, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/B/660b3de1c9994802b0f9769e933b7fa0, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/B/be10d891706146c6a4715cb1cac38f71] into tmpdir=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/.tmp, totalSize=36.9 K 2024-11-18T20:12:17,791 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8d505235538c4e00a2e80fc5deba2fb8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=370, earliestPutTs=1731960734970 2024-11-18T20:12:17,792 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4adf08c5d45f42d895d681c86b015852, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=385, earliestPutTs=1731960737150 2024-11-18T20:12:17,793 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.Compactor(225): Compacting 35279005022e483f8c8b7d07426de443, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=346, earliestPutTs=1731960734920 2024-11-18T20:12:17,793 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.Compactor(225): Compacting 660b3de1c9994802b0f9769e933b7fa0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=370, earliestPutTs=1731960734970 2024-11-18T20:12:17,794 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.Compactor(225): Compacting be10d891706146c6a4715cb1cac38f71, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=385, earliestPutTs=1731960737167 2024-11-18T20:12:17,819 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 326884d1d40acf488e8bd50aea838c73#B#compaction#83 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:12:17,820 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/.tmp/B/45b0482610194430993171686f5e8582 is 50, key is test_row_0/B:col10/1731960737170/Put/seqid=0 2024-11-18T20:12:17,823 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 326884d1d40acf488e8bd50aea838c73#A#compaction#84 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:12:17,824 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/.tmp/A/8727132e5ba546b7b5f9ad74aa3928f8 is 50, key is test_row_0/A:col10/1731960737170/Put/seqid=0 2024-11-18T20:12:17,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(8855): Flush requested on 326884d1d40acf488e8bd50aea838c73 2024-11-18T20:12:17,869 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-18T20:12:17,870 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 326884d1d40acf488e8bd50aea838c73 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-18T20:12:17,883 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073741920_1096 (size=13255) 2024-11-18T20:12:17,891 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35963 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-18T20:12:17,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73. 2024-11-18T20:12:17,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73. as already flushing 2024-11-18T20:12:17,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73. 2024-11-18T20:12:17,892 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 326884d1d40acf488e8bd50aea838c73, NAME => 'TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:12:17,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 326884d1d40acf488e8bd50aea838c73, NAME => 'TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:12:17,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41277 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 326884d1d40acf488e8bd50aea838c73, NAME => 'TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 326884d1d40acf488e8bd50aea838c73, NAME => 'TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:12:17,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-18T20:12:17,897 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:17,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33046 deadline: 1731960797886, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:17,899 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:17,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33028 deadline: 1731960797890, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:17,901 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:17,901 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:17,901 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:12:17,904 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/.tmp/A/8727132e5ba546b7b5f9ad74aa3928f8 as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/A/8727132e5ba546b7b5f9ad74aa3928f8 2024-11-18T20:12:17,905 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:17,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33070 deadline: 1731960797893, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:17,908 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:17,908 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:17,908 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:12:17,908 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:17,909 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:17,909 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:17,910 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:12:17,909 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:17,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33056 deadline: 1731960797896, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:17,910 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:17,910 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:17,910 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:12:17,913 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 326884d1d40acf488e8bd50aea838c73/A of 326884d1d40acf488e8bd50aea838c73 into 8727132e5ba546b7b5f9ad74aa3928f8(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:12:17,914 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 326884d1d40acf488e8bd50aea838c73: 2024-11-18T20:12:17,914 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., storeName=326884d1d40acf488e8bd50aea838c73/A, priority=13, startTime=1731960737786; duration=0sec 2024-11-18T20:12:17,914 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T20:12:17,914 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 326884d1d40acf488e8bd50aea838c73:A 2024-11-18T20:12:17,914 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T20:12:17,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33072 deadline: 1731960797893, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:17,917 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37721 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T20:12:17,918 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HStore(1541): 326884d1d40acf488e8bd50aea838c73/C is initiating minor compaction (all files) 2024-11-18T20:12:17,918 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:17,918 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:17,918 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 326884d1d40acf488e8bd50aea838c73/C in TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73. 2024-11-18T20:12:17,918 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:12:17,918 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/C/f472bbab37374b50b653dca80bbc51ac, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/C/1a4e8da4738b4b9bb9fc00e6ff0346f4, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/C/d318e0d36ef04bbfa81b4a9cec0bed57] into tmpdir=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/.tmp, totalSize=36.8 K 2024-11-18T20:12:17,919 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] compactions.Compactor(225): Compacting f472bbab37374b50b653dca80bbc51ac, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=346, earliestPutTs=1731960734920 2024-11-18T20:12:17,920 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073741921_1097 (size=13255) 2024-11-18T20:12:17,920 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1a4e8da4738b4b9bb9fc00e6ff0346f4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=370, earliestPutTs=1731960734970 2024-11-18T20:12:17,921 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] compactions.Compactor(225): Compacting d318e0d36ef04bbfa81b4a9cec0bed57, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=385, earliestPutTs=1731960737167 2024-11-18T20:12:17,938 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/.tmp/A/822d31fcded2479f9e073562bb3da745 is 50, key is test_row_0/A:col10/1731960737234/Put/seqid=0 2024-11-18T20:12:17,951 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 326884d1d40acf488e8bd50aea838c73#C#compaction#86 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:12:17,952 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/.tmp/C/30143802614f4f2d943e803bb5ada559 is 50, key is test_row_0/C:col10/1731960737170/Put/seqid=0 2024-11-18T20:12:18,015 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073741922_1098 (size=14741) 2024-11-18T20:12:18,019 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:18,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33056 deadline: 1731960798014, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:18,020 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:18,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33070 deadline: 1731960798014, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:18,021 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:18,021 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:18,021 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:18,021 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:12:18,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33028 deadline: 1731960798014, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:18,022 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:18,022 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:18,022 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:18,022 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:18,022 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:12:18,022 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:12:18,029 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:18,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33072 deadline: 1731960798023, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:18,030 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:18,030 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:18,030 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:12:18,036 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073741923_1099 (size=13221) 2024-11-18T20:12:18,047 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35963 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-18T20:12:18,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73. 2024-11-18T20:12:18,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73. as already flushing 2024-11-18T20:12:18,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73. 2024-11-18T20:12:18,047 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 326884d1d40acf488e8bd50aea838c73, NAME => 'TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:12:18,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 326884d1d40acf488e8bd50aea838c73, NAME => 'TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:12:18,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41277 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 326884d1d40acf488e8bd50aea838c73, NAME => 'TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 326884d1d40acf488e8bd50aea838c73, NAME => 'TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:12:18,050 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/.tmp/C/30143802614f4f2d943e803bb5ada559 as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/C/30143802614f4f2d943e803bb5ada559 2024-11-18T20:12:18,058 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 326884d1d40acf488e8bd50aea838c73/C of 326884d1d40acf488e8bd50aea838c73 into 30143802614f4f2d943e803bb5ada559(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:12:18,058 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 326884d1d40acf488e8bd50aea838c73: 2024-11-18T20:12:18,059 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., storeName=326884d1d40acf488e8bd50aea838c73/C, priority=13, startTime=1731960737790; duration=0sec 2024-11-18T20:12:18,059 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:12:18,059 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 326884d1d40acf488e8bd50aea838c73:C 2024-11-18T20:12:18,201 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35963 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-18T20:12:18,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73. 2024-11-18T20:12:18,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73. as already flushing 2024-11-18T20:12:18,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73. 2024-11-18T20:12:18,202 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 326884d1d40acf488e8bd50aea838c73, NAME => 'TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:12:18,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 326884d1d40acf488e8bd50aea838c73, NAME => 'TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:12:18,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-18T20:12:18,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41277 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 326884d1d40acf488e8bd50aea838c73, NAME => 'TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 326884d1d40acf488e8bd50aea838c73, NAME => 'TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:12:18,235 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:18,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33056 deadline: 1731960798233, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:18,236 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:18,236 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:18,236 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:12:18,236 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:18,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33028 deadline: 1731960798234, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:18,237 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:18,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33072 deadline: 1731960798234, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:18,237 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:18,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33070 deadline: 1731960798234, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:18,237 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:18,238 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:18,238 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:12:18,239 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:18,240 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:18,240 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:12:18,240 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:18,240 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:18,240 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:12:18,332 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/.tmp/B/45b0482610194430993171686f5e8582 as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/B/45b0482610194430993171686f5e8582 2024-11-18T20:12:18,343 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 326884d1d40acf488e8bd50aea838c73/B of 326884d1d40acf488e8bd50aea838c73 into 45b0482610194430993171686f5e8582(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:12:18,343 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 326884d1d40acf488e8bd50aea838c73: 2024-11-18T20:12:18,343 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., storeName=326884d1d40acf488e8bd50aea838c73/B, priority=13, startTime=1731960737788; duration=0sec 2024-11-18T20:12:18,343 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:12:18,343 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 326884d1d40acf488e8bd50aea838c73:B 2024-11-18T20:12:18,357 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35963 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-18T20:12:18,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73. 2024-11-18T20:12:18,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73. as already flushing 2024-11-18T20:12:18,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73. 2024-11-18T20:12:18,358 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 326884d1d40acf488e8bd50aea838c73, NAME => 'TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:12:18,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 326884d1d40acf488e8bd50aea838c73, NAME => 'TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:12:18,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41277 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 326884d1d40acf488e8bd50aea838c73, NAME => 'TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 326884d1d40acf488e8bd50aea838c73, NAME => 'TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:12:18,405 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:18,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33046 deadline: 1731960798405, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:18,407 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:18,407 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:18,407 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:12:18,416 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=411 (bloomFilter=true), to=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/.tmp/A/822d31fcded2479f9e073562bb3da745 2024-11-18T20:12:18,439 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/.tmp/B/b4880fc6b7a14c0c9cdf976ce0fd8595 is 50, key is test_row_0/B:col10/1731960737234/Put/seqid=0 2024-11-18T20:12:18,507 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073741924_1100 (size=12301) 2024-11-18T20:12:18,508 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=411 (bloomFilter=true), to=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/.tmp/B/b4880fc6b7a14c0c9cdf976ce0fd8595 2024-11-18T20:12:18,520 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35963 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-18T20:12:18,520 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73. 2024-11-18T20:12:18,521 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73. as already flushing 2024-11-18T20:12:18,521 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73. 2024-11-18T20:12:18,521 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 326884d1d40acf488e8bd50aea838c73, NAME => 'TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:12:18,521 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 326884d1d40acf488e8bd50aea838c73, NAME => 'TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:12:18,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41277 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 326884d1d40acf488e8bd50aea838c73, NAME => 'TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 326884d1d40acf488e8bd50aea838c73, NAME => 'TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:12:18,535 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/.tmp/C/4d511892eb394f2e87f47ae4755c8405 is 50, key is test_row_0/C:col10/1731960737234/Put/seqid=0 2024-11-18T20:12:18,546 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:18,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33056 deadline: 1731960798543, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:18,547 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:18,547 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:18,547 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:12:18,548 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:18,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33072 deadline: 1731960798544, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:18,548 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:18,548 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:18,548 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:12:18,548 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:18,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33028 deadline: 1731960798544, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:18,549 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:18,549 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:18,549 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:12:18,549 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:18,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33070 deadline: 1731960798544, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:18,557 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:18,557 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:18,557 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:12:18,582 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073741925_1101 (size=12301) 2024-11-18T20:12:18,675 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35963 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-18T20:12:18,675 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73. 2024-11-18T20:12:18,675 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73. as already flushing 2024-11-18T20:12:18,675 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73. 2024-11-18T20:12:18,675 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 326884d1d40acf488e8bd50aea838c73, NAME => 'TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:12:18,676 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 326884d1d40acf488e8bd50aea838c73, NAME => 'TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:12:18,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41277 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 326884d1d40acf488e8bd50aea838c73, NAME => 'TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 326884d1d40acf488e8bd50aea838c73, NAME => 'TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:12:18,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-18T20:12:18,829 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35963 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-18T20:12:18,829 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73. 2024-11-18T20:12:18,829 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73. as already flushing 2024-11-18T20:12:18,829 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73. 2024-11-18T20:12:18,829 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 326884d1d40acf488e8bd50aea838c73, NAME => 'TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:12:18,830 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 326884d1d40acf488e8bd50aea838c73, NAME => 'TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:12:18,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41277 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 326884d1d40acf488e8bd50aea838c73, NAME => 'TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 326884d1d40acf488e8bd50aea838c73, NAME => 'TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:12:18,976 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=411 (bloomFilter=true), to=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/.tmp/C/4d511892eb394f2e87f47ae4755c8405 2024-11-18T20:12:18,986 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35963 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-18T20:12:18,986 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73. 2024-11-18T20:12:18,987 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73. as already flushing 2024-11-18T20:12:18,987 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73. 2024-11-18T20:12:18,987 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 326884d1d40acf488e8bd50aea838c73, NAME => 'TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:12:18,987 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 326884d1d40acf488e8bd50aea838c73, NAME => 'TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:12:18,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41277 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 326884d1d40acf488e8bd50aea838c73, NAME => 'TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 326884d1d40acf488e8bd50aea838c73, NAME => 'TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:12:18,989 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/.tmp/A/822d31fcded2479f9e073562bb3da745 as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/A/822d31fcded2479f9e073562bb3da745 2024-11-18T20:12:18,996 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/A/822d31fcded2479f9e073562bb3da745, entries=200, sequenceid=411, filesize=14.4 K 2024-11-18T20:12:18,997 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/.tmp/B/b4880fc6b7a14c0c9cdf976ce0fd8595 as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/B/b4880fc6b7a14c0c9cdf976ce0fd8595 2024-11-18T20:12:19,018 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/B/b4880fc6b7a14c0c9cdf976ce0fd8595, entries=150, sequenceid=411, filesize=12.0 K 2024-11-18T20:12:19,020 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/.tmp/C/4d511892eb394f2e87f47ae4755c8405 as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/C/4d511892eb394f2e87f47ae4755c8405 2024-11-18T20:12:19,027 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/C/4d511892eb394f2e87f47ae4755c8405, entries=150, sequenceid=411, filesize=12.0 K 2024-11-18T20:12:19,028 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for 326884d1d40acf488e8bd50aea838c73 in 1159ms, sequenceid=411, compaction requested=false 2024-11-18T20:12:19,028 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 326884d1d40acf488e8bd50aea838c73: 2024-11-18T20:12:19,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(8855): Flush requested on 326884d1d40acf488e8bd50aea838c73 2024-11-18T20:12:19,056 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-18T20:12:19,056 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 326884d1d40acf488e8bd50aea838c73 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-18T20:12:19,065 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/.tmp/A/403d977ac56a4e4aba816ed73dcc0ea1 is 50, key is test_row_0/A:col10/1731960737884/Put/seqid=0 2024-11-18T20:12:19,080 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073741926_1102 (size=12301) 2024-11-18T20:12:19,114 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:19,115 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:19,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33072 deadline: 1731960799109, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:19,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33070 deadline: 1731960799110, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:19,115 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:19,115 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:19,115 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:12:19,116 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:19,116 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:19,116 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:12:19,119 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:19,119 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:19,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33056 deadline: 1731960799115, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:19,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33028 deadline: 1731960799115, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:19,120 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:19,120 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:19,120 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:12:19,121 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:19,121 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:19,121 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:12:19,144 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35963 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-18T20:12:19,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73. 2024-11-18T20:12:19,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73. as already flushing 2024-11-18T20:12:19,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73. 2024-11-18T20:12:19,144 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 326884d1d40acf488e8bd50aea838c73, NAME => 'TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:12:19,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 326884d1d40acf488e8bd50aea838c73, NAME => 'TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:12:19,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41277 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 326884d1d40acf488e8bd50aea838c73, NAME => 'TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 326884d1d40acf488e8bd50aea838c73, NAME => 'TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:12:19,225 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:19,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33028 deadline: 1731960799224, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:19,225 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:19,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33070 deadline: 1731960799224, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:19,226 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:19,226 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:19,226 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:19,226 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:12:19,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33056 deadline: 1731960799224, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:19,227 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:19,227 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:19,227 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:19,227 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:19,227 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:19,227 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:12:19,227 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:12:19,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33072 deadline: 1731960799225, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:19,227 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:19,227 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:19,227 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:12:19,296 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35963 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-18T20:12:19,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73. 2024-11-18T20:12:19,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73. as already flushing 2024-11-18T20:12:19,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73. 2024-11-18T20:12:19,297 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 326884d1d40acf488e8bd50aea838c73, NAME => 'TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:12:19,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 326884d1d40acf488e8bd50aea838c73, NAME => 'TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:12:19,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41277 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 326884d1d40acf488e8bd50aea838c73, NAME => 'TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 326884d1d40acf488e8bd50aea838c73, NAME => 'TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:12:19,415 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:19,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33046 deadline: 1731960799413, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:19,419 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to c0a89b2656d4,35963,1731960718223 for 'test_row_2' in 326884d1d40acf488e8bd50aea838c73 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2179 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-18T20:12:19,421 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:19,421 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:19,421 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:12:19,436 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:19,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33070 deadline: 1731960799433, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:19,440 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:19,440 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:19,440 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:19,440 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:19,440 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:19,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33056 deadline: 1731960799434, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:19,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33072 deadline: 1731960799434, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:19,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33028 deadline: 1731960799437, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:19,440 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:12:19,441 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:19,441 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:19,441 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:19,441 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:19,441 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:19,441 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:12:19,441 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:19,441 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:12:19,441 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:12:19,450 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35963 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-18T20:12:19,450 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73. 2024-11-18T20:12:19,450 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73. as already flushing 2024-11-18T20:12:19,450 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73. 2024-11-18T20:12:19,450 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 326884d1d40acf488e8bd50aea838c73, NAME => 'TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:12:19,451 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 326884d1d40acf488e8bd50aea838c73, NAME => 'TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:12:19,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41277 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 326884d1d40acf488e8bd50aea838c73, NAME => 'TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 326884d1d40acf488e8bd50aea838c73, NAME => 'TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:12:19,482 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=425 (bloomFilter=true), to=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/.tmp/A/403d977ac56a4e4aba816ed73dcc0ea1 2024-11-18T20:12:19,493 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/.tmp/B/3f239ac7e53a434fbfc59a140b149e5f is 50, key is test_row_0/B:col10/1731960737884/Put/seqid=0 2024-11-18T20:12:19,510 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073741927_1103 (size=12301) 2024-11-18T20:12:19,604 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35963 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-18T20:12:19,604 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73. 2024-11-18T20:12:19,604 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73. as already flushing 2024-11-18T20:12:19,604 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73. 2024-11-18T20:12:19,604 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 326884d1d40acf488e8bd50aea838c73, NAME => 'TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:12:19,604 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 326884d1d40acf488e8bd50aea838c73, NAME => 'TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:12:19,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41277 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 326884d1d40acf488e8bd50aea838c73, NAME => 'TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 326884d1d40acf488e8bd50aea838c73, NAME => 'TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:12:19,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-18T20:12:19,746 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:19,749 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:19,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33028 deadline: 1731960799743, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:19,749 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:19,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33056 deadline: 1731960799744, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:19,750 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:19,750 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:19,750 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:19,750 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:12:19,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33070 deadline: 1731960799744, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:19,754 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:19,754 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:19,754 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:12:19,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33072 deadline: 1731960799743, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:19,755 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:19,755 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:19,755 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:19,755 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:12:19,755 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:19,755 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:12:19,758 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35963 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-18T20:12:19,759 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73. 2024-11-18T20:12:19,759 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73. as already flushing 2024-11-18T20:12:19,759 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73. 2024-11-18T20:12:19,759 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 326884d1d40acf488e8bd50aea838c73, NAME => 'TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:12:19,759 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 326884d1d40acf488e8bd50aea838c73, NAME => 'TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:12:19,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41277 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 326884d1d40acf488e8bd50aea838c73, NAME => 'TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 326884d1d40acf488e8bd50aea838c73, NAME => 'TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:12:19,911 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=425 (bloomFilter=true), to=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/.tmp/B/3f239ac7e53a434fbfc59a140b149e5f 2024-11-18T20:12:19,912 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35963 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-18T20:12:19,912 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73. 2024-11-18T20:12:19,912 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73. as already flushing 2024-11-18T20:12:19,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73. 2024-11-18T20:12:19,913 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 326884d1d40acf488e8bd50aea838c73, NAME => 'TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:12:19,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 326884d1d40acf488e8bd50aea838c73, NAME => 'TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:12:19,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41277 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 326884d1d40acf488e8bd50aea838c73, NAME => 'TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 326884d1d40acf488e8bd50aea838c73, NAME => 'TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:12:19,925 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/.tmp/C/ae65c7a9fa38497ca2777d173a4a27a4 is 50, key is test_row_0/C:col10/1731960737884/Put/seqid=0 2024-11-18T20:12:19,937 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073741928_1104 (size=12301) 2024-11-18T20:12:19,939 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=425 (bloomFilter=true), to=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/.tmp/C/ae65c7a9fa38497ca2777d173a4a27a4 2024-11-18T20:12:19,949 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/.tmp/A/403d977ac56a4e4aba816ed73dcc0ea1 as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/A/403d977ac56a4e4aba816ed73dcc0ea1 2024-11-18T20:12:19,959 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/A/403d977ac56a4e4aba816ed73dcc0ea1, entries=150, sequenceid=425, filesize=12.0 K 2024-11-18T20:12:19,960 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/.tmp/B/3f239ac7e53a434fbfc59a140b149e5f as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/B/3f239ac7e53a434fbfc59a140b149e5f 2024-11-18T20:12:19,974 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/B/3f239ac7e53a434fbfc59a140b149e5f, entries=150, sequenceid=425, filesize=12.0 K 2024-11-18T20:12:19,975 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/.tmp/C/ae65c7a9fa38497ca2777d173a4a27a4 as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/C/ae65c7a9fa38497ca2777d173a4a27a4 2024-11-18T20:12:19,983 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/C/ae65c7a9fa38497ca2777d173a4a27a4, entries=150, sequenceid=425, filesize=12.0 K 2024-11-18T20:12:19,984 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=154.31 KB/158010 for 326884d1d40acf488e8bd50aea838c73 in 928ms, sequenceid=425, compaction requested=true 2024-11-18T20:12:19,984 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 326884d1d40acf488e8bd50aea838c73: 2024-11-18T20:12:19,984 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 326884d1d40acf488e8bd50aea838c73:A, priority=-2147483648, current under compaction store size is 1 2024-11-18T20:12:19,984 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:12:19,984 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T20:12:19,984 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 326884d1d40acf488e8bd50aea838c73:B, priority=-2147483648, current under compaction store size is 2 2024-11-18T20:12:19,984 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:12:19,984 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T20:12:19,984 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 326884d1d40acf488e8bd50aea838c73:C, priority=-2147483648, current under compaction store size is 3 2024-11-18T20:12:19,984 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T20:12:19,986 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37857 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T20:12:19,986 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40297 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T20:12:19,986 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HStore(1541): 326884d1d40acf488e8bd50aea838c73/B is initiating minor compaction (all files) 2024-11-18T20:12:19,986 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HStore(1541): 326884d1d40acf488e8bd50aea838c73/A is initiating minor compaction (all files) 2024-11-18T20:12:19,986 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 326884d1d40acf488e8bd50aea838c73/B in TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73. 2024-11-18T20:12:19,986 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 326884d1d40acf488e8bd50aea838c73/A in TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73. 2024-11-18T20:12:19,986 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/A/8727132e5ba546b7b5f9ad74aa3928f8, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/A/822d31fcded2479f9e073562bb3da745, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/A/403d977ac56a4e4aba816ed73dcc0ea1] into tmpdir=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/.tmp, totalSize=39.4 K 2024-11-18T20:12:19,986 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/B/45b0482610194430993171686f5e8582, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/B/b4880fc6b7a14c0c9cdf976ce0fd8595, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/B/3f239ac7e53a434fbfc59a140b149e5f] into tmpdir=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/.tmp, totalSize=37.0 K 2024-11-18T20:12:19,987 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.Compactor(225): Compacting 45b0482610194430993171686f5e8582, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=385, earliestPutTs=1731960737167 2024-11-18T20:12:19,987 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8727132e5ba546b7b5f9ad74aa3928f8, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=385, earliestPutTs=1731960737167 2024-11-18T20:12:19,987 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.Compactor(225): Compacting b4880fc6b7a14c0c9cdf976ce0fd8595, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=411, earliestPutTs=1731960737234 2024-11-18T20:12:19,988 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] compactions.Compactor(225): Compacting 822d31fcded2479f9e073562bb3da745, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=411, earliestPutTs=1731960737231 2024-11-18T20:12:19,988 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.Compactor(225): Compacting 3f239ac7e53a434fbfc59a140b149e5f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=425, earliestPutTs=1731960737884 2024-11-18T20:12:19,988 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] compactions.Compactor(225): Compacting 403d977ac56a4e4aba816ed73dcc0ea1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=425, earliestPutTs=1731960737884 2024-11-18T20:12:19,999 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 326884d1d40acf488e8bd50aea838c73#A#compaction#92 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:12:19,999 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/.tmp/A/7030952e566349e99f3949ae7842129c is 50, key is test_row_0/A:col10/1731960737884/Put/seqid=0 2024-11-18T20:12:20,007 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 326884d1d40acf488e8bd50aea838c73#B#compaction#93 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:12:20,008 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/.tmp/B/d0d1139f7ebe4351acba24449ed94fc4 is 50, key is test_row_0/B:col10/1731960737884/Put/seqid=0 2024-11-18T20:12:20,017 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073741930_1106 (size=13357) 2024-11-18T20:12:20,039 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073741929_1105 (size=13357) 2024-11-18T20:12:20,046 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/.tmp/A/7030952e566349e99f3949ae7842129c as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/A/7030952e566349e99f3949ae7842129c 2024-11-18T20:12:20,065 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 326884d1d40acf488e8bd50aea838c73/A of 326884d1d40acf488e8bd50aea838c73 into 7030952e566349e99f3949ae7842129c(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:12:20,065 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 326884d1d40acf488e8bd50aea838c73: 2024-11-18T20:12:20,065 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., storeName=326884d1d40acf488e8bd50aea838c73/A, priority=13, startTime=1731960739984; duration=0sec 2024-11-18T20:12:20,065 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T20:12:20,065 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 326884d1d40acf488e8bd50aea838c73:A 2024-11-18T20:12:20,065 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T20:12:20,067 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35963 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-18T20:12:20,067 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73. 2024-11-18T20:12:20,067 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2902): Flushing 326884d1d40acf488e8bd50aea838c73 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-18T20:12:20,069 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37823 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T20:12:20,069 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HStore(1541): 326884d1d40acf488e8bd50aea838c73/C is initiating minor compaction (all files) 2024-11-18T20:12:20,069 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 326884d1d40acf488e8bd50aea838c73/C in TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73. 2024-11-18T20:12:20,069 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/C/30143802614f4f2d943e803bb5ada559, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/C/4d511892eb394f2e87f47ae4755c8405, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/C/ae65c7a9fa38497ca2777d173a4a27a4] into tmpdir=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/.tmp, totalSize=36.9 K 2024-11-18T20:12:20,071 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] compactions.Compactor(225): Compacting 30143802614f4f2d943e803bb5ada559, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=385, earliestPutTs=1731960737167 2024-11-18T20:12:20,071 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4d511892eb394f2e87f47ae4755c8405, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=411, earliestPutTs=1731960737234 2024-11-18T20:12:20,071 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] compactions.Compactor(225): Compacting ae65c7a9fa38497ca2777d173a4a27a4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=425, earliestPutTs=1731960737884 2024-11-18T20:12:20,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/.tmp/A/4b796b62349740489ebc01ceb833f45e is 50, key is test_row_0/A:col10/1731960739105/Put/seqid=0 2024-11-18T20:12:20,083 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 326884d1d40acf488e8bd50aea838c73#C#compaction#95 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:12:20,084 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/.tmp/C/0bf801180970467380d807e7f4462a63 is 50, key is test_row_0/C:col10/1731960737884/Put/seqid=0 2024-11-18T20:12:20,117 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073741932_1108 (size=13323) 2024-11-18T20:12:20,121 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073741931_1107 (size=12301) 2024-11-18T20:12:20,126 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=452 (bloomFilter=true), to=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/.tmp/A/4b796b62349740489ebc01ceb833f45e 2024-11-18T20:12:20,131 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/.tmp/C/0bf801180970467380d807e7f4462a63 as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/C/0bf801180970467380d807e7f4462a63 2024-11-18T20:12:20,146 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 326884d1d40acf488e8bd50aea838c73/C of 326884d1d40acf488e8bd50aea838c73 into 0bf801180970467380d807e7f4462a63(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:12:20,146 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 326884d1d40acf488e8bd50aea838c73: 2024-11-18T20:12:20,146 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., storeName=326884d1d40acf488e8bd50aea838c73/C, priority=13, startTime=1731960739984; duration=0sec 2024-11-18T20:12:20,146 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:12:20,146 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 326884d1d40acf488e8bd50aea838c73:C 2024-11-18T20:12:20,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/.tmp/B/0fb361e5aff54623bc5860937e9c8f58 is 50, key is test_row_0/B:col10/1731960739105/Put/seqid=0 2024-11-18T20:12:20,179 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073741933_1109 (size=12301) 2024-11-18T20:12:20,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(8855): Flush requested on 326884d1d40acf488e8bd50aea838c73 2024-11-18T20:12:20,256 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73. as already flushing 2024-11-18T20:12:20,275 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:20,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33072 deadline: 1731960800271, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:20,276 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:20,276 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:20,276 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:20,276 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:12:20,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33070 deadline: 1731960800272, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:20,277 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:20,277 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:20,277 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:20,277 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:12:20,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33028 deadline: 1731960800272, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:20,277 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:20,277 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:20,277 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:12:20,280 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:20,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33056 deadline: 1731960800274, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:20,281 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:20,281 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:20,281 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:12:20,389 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:20,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33028 deadline: 1731960800384, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:20,390 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:20,390 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:20,390 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:20,390 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:12:20,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33070 deadline: 1731960800384, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:20,390 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:20,390 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:20,390 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:12:20,391 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:20,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33056 deadline: 1731960800384, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:20,391 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:20,391 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:20,392 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:12:20,392 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:20,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33072 deadline: 1731960800384, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:20,392 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:20,393 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:20,393 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:12:20,430 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/.tmp/B/d0d1139f7ebe4351acba24449ed94fc4 as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/B/d0d1139f7ebe4351acba24449ed94fc4 2024-11-18T20:12:20,442 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 326884d1d40acf488e8bd50aea838c73/B of 326884d1d40acf488e8bd50aea838c73 into d0d1139f7ebe4351acba24449ed94fc4(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:12:20,442 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 326884d1d40acf488e8bd50aea838c73: 2024-11-18T20:12:20,442 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., storeName=326884d1d40acf488e8bd50aea838c73/B, priority=13, startTime=1731960739984; duration=0sec 2024-11-18T20:12:20,442 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:12:20,445 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 326884d1d40acf488e8bd50aea838c73:B 2024-11-18T20:12:20,580 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=452 (bloomFilter=true), to=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/.tmp/B/0fb361e5aff54623bc5860937e9c8f58 2024-11-18T20:12:20,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/.tmp/C/234cc462460544368a9fdfd2b1c17cfc is 50, key is test_row_0/C:col10/1731960739105/Put/seqid=0 2024-11-18T20:12:20,594 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:20,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33028 deadline: 1731960800593, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:20,595 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:20,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33070 deadline: 1731960800593, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:20,595 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:20,595 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:20,595 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:12:20,597 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:20,597 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:20,597 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:12:20,605 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:20,605 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:20,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33056 deadline: 1731960800603, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:20,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33072 deadline: 1731960800603, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:20,606 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:20,606 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:20,606 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:12:20,606 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:20,606 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:20,606 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:12:20,638 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073741934_1110 (size=12301) 2024-11-18T20:12:20,905 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:20,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33070 deadline: 1731960800903, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:20,906 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:20,906 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:20,906 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:12:20,907 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:20,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33028 deadline: 1731960800903, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:20,908 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:20,908 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:20,908 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:12:20,914 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:20,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33072 deadline: 1731960800913, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:20,915 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:20,915 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:20,915 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:20,915 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:12:20,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33056 deadline: 1731960800913, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:20,915 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:20,915 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:20,915 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:12:21,037 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=452 (bloomFilter=true), to=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/.tmp/C/234cc462460544368a9fdfd2b1c17cfc 2024-11-18T20:12:21,044 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/.tmp/A/4b796b62349740489ebc01ceb833f45e as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/A/4b796b62349740489ebc01ceb833f45e 2024-11-18T20:12:21,056 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/A/4b796b62349740489ebc01ceb833f45e, entries=150, sequenceid=452, filesize=12.0 K 2024-11-18T20:12:21,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/.tmp/B/0fb361e5aff54623bc5860937e9c8f58 as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/B/0fb361e5aff54623bc5860937e9c8f58 2024-11-18T20:12:21,065 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/B/0fb361e5aff54623bc5860937e9c8f58, entries=150, sequenceid=452, filesize=12.0 K 2024-11-18T20:12:21,069 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/.tmp/C/234cc462460544368a9fdfd2b1c17cfc as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/C/234cc462460544368a9fdfd2b1c17cfc 2024-11-18T20:12:21,080 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/C/234cc462460544368a9fdfd2b1c17cfc, entries=150, sequenceid=452, filesize=12.0 K 2024-11-18T20:12:21,081 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for 326884d1d40acf488e8bd50aea838c73 in 1014ms, sequenceid=452, compaction requested=false 2024-11-18T20:12:21,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2603): Flush status journal for 326884d1d40acf488e8bd50aea838c73: 2024-11-18T20:12:21,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73. 2024-11-18T20:12:21,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=22 2024-11-18T20:12:21,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41277 {}] master.HMaster(4169): Remote procedure done, pid=22 2024-11-18T20:12:21,086 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=22, resume processing ppid=21 2024-11-18T20:12:21,087 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=22, ppid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.5030 sec 2024-11-18T20:12:21,090 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees in 3.5130 sec 2024-11-18T20:12:21,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(8855): Flush requested on 326884d1d40acf488e8bd50aea838c73 2024-11-18T20:12:21,415 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-18T20:12:21,415 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 326884d1d40acf488e8bd50aea838c73 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-18T20:12:21,427 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/.tmp/A/db998326449549ef9212fd525c6b6217 is 50, key is test_row_0/A:col10/1731960740272/Put/seqid=0 2024-11-18T20:12:21,459 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073741935_1111 (size=14741) 2024-11-18T20:12:21,460 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=465 (bloomFilter=true), to=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/.tmp/A/db998326449549ef9212fd525c6b6217 2024-11-18T20:12:21,477 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:21,477 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:21,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33070 deadline: 1731960801471, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:21,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33028 deadline: 1731960801467, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:21,477 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:21,477 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:21,477 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:21,478 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:12:21,478 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:21,478 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:12:21,480 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:21,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33046 deadline: 1731960801476, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:21,481 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:21,481 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:21,481 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:12:21,482 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:21,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33072 deadline: 1731960801478, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:21,482 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/.tmp/B/a5db33483c20489da7afa0ff39f1ffe3 is 50, key is test_row_0/B:col10/1731960740272/Put/seqid=0 2024-11-18T20:12:21,482 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:21,482 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:21,482 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:12:21,482 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:21,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33056 deadline: 1731960801478, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:21,483 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:21,483 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:21,483 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:12:21,498 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073741936_1112 (size=12301) 2024-11-18T20:12:21,504 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=465 (bloomFilter=true), to=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/.tmp/B/a5db33483c20489da7afa0ff39f1ffe3 2024-11-18T20:12:21,520 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/.tmp/C/14c7cc34aa9f4feb9d610a1b8c652b4c is 50, key is test_row_0/C:col10/1731960740272/Put/seqid=0 2024-11-18T20:12:21,534 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073741937_1113 (size=12301) 2024-11-18T20:12:21,585 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:21,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33028 deadline: 1731960801584, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:21,586 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:21,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33070 deadline: 1731960801584, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:21,587 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:21,587 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:21,587 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:21,587 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:21,587 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:12:21,587 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:12:21,597 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:21,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33046 deadline: 1731960801593, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:21,598 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:21,598 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:21,598 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:21,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33072 deadline: 1731960801594, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:21,598 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:12:21,598 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:21,598 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:21,599 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:12:21,599 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:21,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33056 deadline: 1731960801594, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:21,600 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:21,600 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:21,600 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:12:21,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-18T20:12:21,744 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-18T20:12:21,745 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41277 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-18T20:12:21,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41277 {}] procedure2.ProcedureExecutor(1139): Stored pid=23, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees 2024-11-18T20:12:21,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-18T20:12:21,750 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=23, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-18T20:12:21,751 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=23, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-18T20:12:21,751 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=24, ppid=23, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-18T20:12:21,801 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:21,801 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:21,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33070 deadline: 1731960801793, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:21,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33028 deadline: 1731960801793, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:21,802 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:21,802 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:21,802 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:21,802 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:21,802 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:12:21,802 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:12:21,806 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:21,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33046 deadline: 1731960801803, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:21,807 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:21,807 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:21,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33072 deadline: 1731960801803, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:21,807 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:21,807 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:12:21,807 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:21,807 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:21,807 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:12:21,808 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:21,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33056 deadline: 1731960801803, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:21,808 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:21,808 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:21,808 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:12:21,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-18T20:12:21,905 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35963 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-18T20:12:21,906 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73. 2024-11-18T20:12:21,906 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73. as already flushing 2024-11-18T20:12:21,906 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73. 2024-11-18T20:12:21,906 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => 326884d1d40acf488e8bd50aea838c73, NAME => 'TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:12:21,906 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => 326884d1d40acf488e8bd50aea838c73, NAME => 'TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:12:21,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41277 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 326884d1d40acf488e8bd50aea838c73, NAME => 'TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 326884d1d40acf488e8bd50aea838c73, NAME => 'TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:12:21,935 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=465 (bloomFilter=true), to=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/.tmp/C/14c7cc34aa9f4feb9d610a1b8c652b4c 2024-11-18T20:12:21,949 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/.tmp/A/db998326449549ef9212fd525c6b6217 as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/A/db998326449549ef9212fd525c6b6217 2024-11-18T20:12:21,973 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/A/db998326449549ef9212fd525c6b6217, entries=200, sequenceid=465, filesize=14.4 K 2024-11-18T20:12:21,976 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/.tmp/B/a5db33483c20489da7afa0ff39f1ffe3 as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/B/a5db33483c20489da7afa0ff39f1ffe3 2024-11-18T20:12:21,991 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/B/a5db33483c20489da7afa0ff39f1ffe3, entries=150, sequenceid=465, filesize=12.0 K 2024-11-18T20:12:21,993 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/.tmp/C/14c7cc34aa9f4feb9d610a1b8c652b4c as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/C/14c7cc34aa9f4feb9d610a1b8c652b4c 2024-11-18T20:12:22,008 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/C/14c7cc34aa9f4feb9d610a1b8c652b4c, entries=150, sequenceid=465, filesize=12.0 K 2024-11-18T20:12:22,009 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 326884d1d40acf488e8bd50aea838c73 in 594ms, sequenceid=465, compaction requested=true 2024-11-18T20:12:22,009 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 326884d1d40acf488e8bd50aea838c73: 2024-11-18T20:12:22,010 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 326884d1d40acf488e8bd50aea838c73:A, priority=-2147483648, current under compaction store size is 1 2024-11-18T20:12:22,010 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:12:22,010 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T20:12:22,010 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 326884d1d40acf488e8bd50aea838c73:B, priority=-2147483648, current under compaction store size is 2 2024-11-18T20:12:22,010 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T20:12:22,010 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:12:22,010 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 326884d1d40acf488e8bd50aea838c73:C, priority=-2147483648, current under compaction store size is 3 2024-11-18T20:12:22,010 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T20:12:22,013 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37959 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T20:12:22,013 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HStore(1541): 326884d1d40acf488e8bd50aea838c73/B is initiating minor compaction (all files) 2024-11-18T20:12:22,013 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 326884d1d40acf488e8bd50aea838c73/B in TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73. 2024-11-18T20:12:22,013 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/B/d0d1139f7ebe4351acba24449ed94fc4, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/B/0fb361e5aff54623bc5860937e9c8f58, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/B/a5db33483c20489da7afa0ff39f1ffe3] into tmpdir=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/.tmp, totalSize=37.1 K 2024-11-18T20:12:22,013 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40399 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T20:12:22,014 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HStore(1541): 326884d1d40acf488e8bd50aea838c73/A is initiating minor compaction (all files) 2024-11-18T20:12:22,014 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 326884d1d40acf488e8bd50aea838c73/A in TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73. 2024-11-18T20:12:22,014 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/A/7030952e566349e99f3949ae7842129c, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/A/4b796b62349740489ebc01ceb833f45e, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/A/db998326449549ef9212fd525c6b6217] into tmpdir=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/.tmp, totalSize=39.5 K 2024-11-18T20:12:22,014 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.Compactor(225): Compacting d0d1139f7ebe4351acba24449ed94fc4, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=425, earliestPutTs=1731960737884 2024-11-18T20:12:22,014 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7030952e566349e99f3949ae7842129c, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=425, earliestPutTs=1731960737884 2024-11-18T20:12:22,016 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.Compactor(225): Compacting 0fb361e5aff54623bc5860937e9c8f58, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=452, earliestPutTs=1731960739105 2024-11-18T20:12:22,017 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4b796b62349740489ebc01ceb833f45e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=452, earliestPutTs=1731960739105 2024-11-18T20:12:22,018 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.Compactor(225): Compacting a5db33483c20489da7afa0ff39f1ffe3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=465, earliestPutTs=1731960740255 2024-11-18T20:12:22,018 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] compactions.Compactor(225): Compacting db998326449549ef9212fd525c6b6217, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=465, earliestPutTs=1731960740255 2024-11-18T20:12:22,034 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 326884d1d40acf488e8bd50aea838c73#A#compaction#101 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:12:22,035 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/.tmp/A/90f6af9cf92d4df293c47c65a5aa33e5 is 50, key is test_row_0/A:col10/1731960740272/Put/seqid=0 2024-11-18T20:12:22,043 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 326884d1d40acf488e8bd50aea838c73#B#compaction#102 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:12:22,044 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/.tmp/B/952e1a9c24c7450cb645c7adeaf61807 is 50, key is test_row_0/B:col10/1731960740272/Put/seqid=0 2024-11-18T20:12:22,059 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073741939_1115 (size=13459) 2024-11-18T20:12:22,060 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35963 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-18T20:12:22,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73. 2024-11-18T20:12:22,060 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2902): Flushing 326884d1d40acf488e8bd50aea838c73 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-18T20:12:22,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-18T20:12:22,076 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/.tmp/B/952e1a9c24c7450cb645c7adeaf61807 as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/B/952e1a9c24c7450cb645c7adeaf61807 2024-11-18T20:12:22,086 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 326884d1d40acf488e8bd50aea838c73/B of 326884d1d40acf488e8bd50aea838c73 into 952e1a9c24c7450cb645c7adeaf61807(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:12:22,086 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 326884d1d40acf488e8bd50aea838c73: 2024-11-18T20:12:22,086 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., storeName=326884d1d40acf488e8bd50aea838c73/B, priority=13, startTime=1731960742010; duration=0sec 2024-11-18T20:12:22,086 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T20:12:22,086 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 326884d1d40acf488e8bd50aea838c73:B 2024-11-18T20:12:22,086 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T20:12:22,089 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37925 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T20:12:22,090 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HStore(1541): 326884d1d40acf488e8bd50aea838c73/C is initiating minor compaction (all files) 2024-11-18T20:12:22,090 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 326884d1d40acf488e8bd50aea838c73/C in TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73. 2024-11-18T20:12:22,090 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/C/0bf801180970467380d807e7f4462a63, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/C/234cc462460544368a9fdfd2b1c17cfc, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/C/14c7cc34aa9f4feb9d610a1b8c652b4c] into tmpdir=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/.tmp, totalSize=37.0 K 2024-11-18T20:12:22,090 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.Compactor(225): Compacting 0bf801180970467380d807e7f4462a63, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=425, earliestPutTs=1731960737884 2024-11-18T20:12:22,091 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.Compactor(225): Compacting 234cc462460544368a9fdfd2b1c17cfc, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=452, earliestPutTs=1731960739105 2024-11-18T20:12:22,091 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.Compactor(225): Compacting 14c7cc34aa9f4feb9d610a1b8c652b4c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=465, earliestPutTs=1731960740255 2024-11-18T20:12:22,102 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073741938_1114 (size=13459) 2024-11-18T20:12:22,115 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/.tmp/A/74369a8e70224776ae6b6ad11688cd1d is 50, key is test_row_0/A:col10/1731960741475/Put/seqid=0 2024-11-18T20:12:22,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(8855): Flush requested on 326884d1d40acf488e8bd50aea838c73 2024-11-18T20:12:22,118 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73. as already flushing 2024-11-18T20:12:22,124 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 326884d1d40acf488e8bd50aea838c73#C#compaction#104 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:12:22,125 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/.tmp/C/79a7f500da114675bd904acea264b4fe is 50, key is test_row_0/C:col10/1731960740272/Put/seqid=0 2024-11-18T20:12:22,129 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:22,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33072 deadline: 1731960802125, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:22,129 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:22,129 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:22,129 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:12:22,130 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:22,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33028 deadline: 1731960802126, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:22,130 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:22,130 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:22,130 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:12:22,135 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:22,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33070 deadline: 1731960802128, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:22,135 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:22,135 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:22,135 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:22,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33046 deadline: 1731960802130, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:22,135 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:12:22,136 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:22,136 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:22,136 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:22,136 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:12:22,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33056 deadline: 1731960802130, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:22,137 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:22,137 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:22,137 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:12:22,171 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073741940_1116 (size=12301) 2024-11-18T20:12:22,172 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=490 (bloomFilter=true), to=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/.tmp/A/74369a8e70224776ae6b6ad11688cd1d 2024-11-18T20:12:22,187 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/.tmp/B/5eb29e1f83e740efbdef5a43c9f0dd9c is 50, key is test_row_0/B:col10/1731960741475/Put/seqid=0 2024-11-18T20:12:22,194 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073741941_1117 (size=13425) 2024-11-18T20:12:22,201 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073741942_1118 (size=12301) 2024-11-18T20:12:22,205 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/.tmp/C/79a7f500da114675bd904acea264b4fe as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/C/79a7f500da114675bd904acea264b4fe 2024-11-18T20:12:22,234 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:22,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33028 deadline: 1731960802233, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:22,235 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:22,235 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:22,235 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:22,235 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:12:22,237 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 326884d1d40acf488e8bd50aea838c73/C of 326884d1d40acf488e8bd50aea838c73 into 79a7f500da114675bd904acea264b4fe(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:12:22,237 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 326884d1d40acf488e8bd50aea838c73: 2024-11-18T20:12:22,237 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., storeName=326884d1d40acf488e8bd50aea838c73/C, priority=13, startTime=1731960742010; duration=0sec 2024-11-18T20:12:22,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33072 deadline: 1731960802233, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:22,249 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:12:22,249 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 326884d1d40acf488e8bd50aea838c73:C 2024-11-18T20:12:22,249 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:22,250 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:22,250 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:12:22,257 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:22,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33056 deadline: 1731960802247, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:22,258 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:22,258 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:22,258 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:12:22,257 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:22,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33046 deadline: 1731960802247, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:22,262 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:22,262 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:22,262 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:12:22,265 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:22,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33070 deadline: 1731960802248, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:22,266 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:22,266 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:22,266 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:12:22,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-18T20:12:22,443 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:22,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33028 deadline: 1731960802443, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:22,444 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:22,444 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:22,444 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:12:22,454 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:22,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33072 deadline: 1731960802453, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:22,455 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:22,455 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:22,455 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:12:22,465 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:22,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33056 deadline: 1731960802463, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:22,465 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:22,465 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:22,466 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:12:22,476 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:22,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33070 deadline: 1731960802473, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:22,476 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:22,476 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:22,476 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:22,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33046 deadline: 1731960802473, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:22,477 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:12:22,481 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:22,481 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:22,481 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:12:22,514 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/.tmp/A/90f6af9cf92d4df293c47c65a5aa33e5 as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/A/90f6af9cf92d4df293c47c65a5aa33e5 2024-11-18T20:12:22,531 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 326884d1d40acf488e8bd50aea838c73/A of 326884d1d40acf488e8bd50aea838c73 into 90f6af9cf92d4df293c47c65a5aa33e5(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:12:22,532 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 326884d1d40acf488e8bd50aea838c73: 2024-11-18T20:12:22,532 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., storeName=326884d1d40acf488e8bd50aea838c73/A, priority=13, startTime=1731960742010; duration=0sec 2024-11-18T20:12:22,532 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:12:22,532 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 326884d1d40acf488e8bd50aea838c73:A 2024-11-18T20:12:22,602 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=490 (bloomFilter=true), to=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/.tmp/B/5eb29e1f83e740efbdef5a43c9f0dd9c 2024-11-18T20:12:22,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/.tmp/C/6a4ad8aeebf541e3a78afbb395b5e0e3 is 50, key is test_row_0/C:col10/1731960741475/Put/seqid=0 2024-11-18T20:12:22,627 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073741943_1119 (size=12301) 2024-11-18T20:12:22,628 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=490 (bloomFilter=true), to=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/.tmp/C/6a4ad8aeebf541e3a78afbb395b5e0e3 2024-11-18T20:12:22,639 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/.tmp/A/74369a8e70224776ae6b6ad11688cd1d as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/A/74369a8e70224776ae6b6ad11688cd1d 2024-11-18T20:12:22,649 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/A/74369a8e70224776ae6b6ad11688cd1d, entries=150, sequenceid=490, filesize=12.0 K 2024-11-18T20:12:22,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/.tmp/B/5eb29e1f83e740efbdef5a43c9f0dd9c as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/B/5eb29e1f83e740efbdef5a43c9f0dd9c 2024-11-18T20:12:22,660 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/B/5eb29e1f83e740efbdef5a43c9f0dd9c, entries=150, sequenceid=490, filesize=12.0 K 2024-11-18T20:12:22,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/.tmp/C/6a4ad8aeebf541e3a78afbb395b5e0e3 as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/C/6a4ad8aeebf541e3a78afbb395b5e0e3 2024-11-18T20:12:22,671 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/C/6a4ad8aeebf541e3a78afbb395b5e0e3, entries=150, sequenceid=490, filesize=12.0 K 2024-11-18T20:12:22,671 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.StoreScanner(997): StoreScanner already has the close lock. There is no need to updateReaders 2024-11-18T20:12:22,672 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=60.38 KB/61830 for 326884d1d40acf488e8bd50aea838c73 in 612ms, sequenceid=490, compaction requested=false 2024-11-18T20:12:22,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2603): Flush status journal for 326884d1d40acf488e8bd50aea838c73: 2024-11-18T20:12:22,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73. 2024-11-18T20:12:22,673 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=24 2024-11-18T20:12:22,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41277 {}] master.HMaster(4169): Remote procedure done, pid=24 2024-11-18T20:12:22,678 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=24, resume processing ppid=23 2024-11-18T20:12:22,678 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=24, ppid=23, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 925 msec 2024-11-18T20:12:22,684 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=23, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees in 937 msec 2024-11-18T20:12:22,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(8855): Flush requested on 326884d1d40acf488e8bd50aea838c73 2024-11-18T20:12:22,755 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-18T20:12:22,755 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 326884d1d40acf488e8bd50aea838c73 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-18T20:12:22,768 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/.tmp/A/297625051e604ebc8357f0b03c23a862 is 50, key is test_row_0/A:col10/1731960742125/Put/seqid=0 2024-11-18T20:12:22,794 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073741944_1120 (size=12301) 2024-11-18T20:12:22,807 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:22,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33028 deadline: 1731960802801, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:22,808 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:22,808 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:22,808 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:22,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33046 deadline: 1731960802803, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:22,808 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:12:22,808 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:22,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33056 deadline: 1731960802803, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:22,808 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:22,808 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:22,808 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:12:22,809 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:22,809 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:22,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33070 deadline: 1731960802804, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:22,809 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:22,809 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:12:22,809 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:22,809 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:22,809 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:12:22,812 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:22,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33072 deadline: 1731960802808, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:22,818 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:22,818 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:22,819 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:12:22,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-18T20:12:22,893 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-18T20:12:22,894 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41277 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-18T20:12:22,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41277 {}] procedure2.ProcedureExecutor(1139): Stored pid=25, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees 2024-11-18T20:12:22,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-18T20:12:22,896 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=25, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-18T20:12:22,897 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=25, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-18T20:12:22,897 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=26, ppid=25, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-18T20:12:22,914 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:22,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33028 deadline: 1731960802914, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:22,915 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:22,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33056 deadline: 1731960802914, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:22,915 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:22,915 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:22,915 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:22,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33046 deadline: 1731960802915, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:22,915 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:12:22,916 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:22,916 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:22,916 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:12:22,916 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:22,916 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:22,916 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:12:22,917 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:22,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33070 deadline: 1731960802917, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:22,918 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:22,918 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:22,918 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:12:22,926 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:22,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33072 deadline: 1731960802923, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:22,927 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:22,928 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:22,928 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:12:23,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-18T20:12:23,050 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35963 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-18T20:12:23,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73. 2024-11-18T20:12:23,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73. as already flushing 2024-11-18T20:12:23,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73. 2024-11-18T20:12:23,050 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => 326884d1d40acf488e8bd50aea838c73, NAME => 'TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:12:23,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => 326884d1d40acf488e8bd50aea838c73, NAME => 'TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:12:23,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41277 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 326884d1d40acf488e8bd50aea838c73, NAME => 'TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 326884d1d40acf488e8bd50aea838c73, NAME => 'TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:12:23,125 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:23,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33046 deadline: 1731960803124, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:23,126 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:23,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33070 deadline: 1731960803124, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:23,127 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:23,127 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:23,127 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:12:23,128 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:23,128 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:23,128 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:12:23,134 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:23,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33056 deadline: 1731960803133, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:23,134 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:23,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33028 deadline: 1731960803133, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:23,134 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:23,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33072 deadline: 1731960803133, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:23,135 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:23,135 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:23,135 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:12:23,135 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:23,135 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:23,135 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:12:23,135 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:23,136 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:23,136 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:12:23,192 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=506 (bloomFilter=true), to=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/.tmp/A/297625051e604ebc8357f0b03c23a862 2024-11-18T20:12:23,204 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35963 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-18T20:12:23,204 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/.tmp/B/f2e3abbc14ba4c25a89bd3bb426ad544 is 50, key is test_row_0/B:col10/1731960742125/Put/seqid=0 2024-11-18T20:12:23,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73. 2024-11-18T20:12:23,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73. as already flushing 2024-11-18T20:12:23,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73. 2024-11-18T20:12:23,205 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => 326884d1d40acf488e8bd50aea838c73, NAME => 'TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:12:23,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => 326884d1d40acf488e8bd50aea838c73, NAME => 'TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:12:23,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41277 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 326884d1d40acf488e8bd50aea838c73, NAME => 'TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 326884d1d40acf488e8bd50aea838c73, NAME => 'TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:12:23,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-18T20:12:23,219 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073741945_1121 (size=12301) 2024-11-18T20:12:23,358 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35963 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-18T20:12:23,359 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73. 2024-11-18T20:12:23,359 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73. as already flushing 2024-11-18T20:12:23,359 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73. 2024-11-18T20:12:23,359 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => 326884d1d40acf488e8bd50aea838c73, NAME => 'TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:12:23,359 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => 326884d1d40acf488e8bd50aea838c73, NAME => 'TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:12:23,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41277 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 326884d1d40acf488e8bd50aea838c73, NAME => 'TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 326884d1d40acf488e8bd50aea838c73, NAME => 'TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:12:23,435 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:23,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33046 deadline: 1731960803433, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:23,436 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:23,436 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:23,436 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:23,436 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:12:23,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33070 deadline: 1731960803433, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:23,437 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:23,437 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:23,437 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:12:23,444 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:23,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33028 deadline: 1731960803443, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:23,445 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:23,445 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:23,445 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:23,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33072 deadline: 1731960803443, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:23,445 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:12:23,445 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:23,445 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:23,445 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:23,445 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:12:23,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33056 deadline: 1731960803443, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:23,446 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:23,446 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:23,446 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:12:23,513 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35963 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-18T20:12:23,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73. 2024-11-18T20:12:23,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73. as already flushing 2024-11-18T20:12:23,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73. 2024-11-18T20:12:23,514 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => 326884d1d40acf488e8bd50aea838c73, NAME => 'TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:12:23,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => 326884d1d40acf488e8bd50aea838c73, NAME => 'TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:12:23,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41277 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 326884d1d40acf488e8bd50aea838c73, NAME => 'TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 326884d1d40acf488e8bd50aea838c73, NAME => 'TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:12:23,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-18T20:12:23,620 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=506 (bloomFilter=true), to=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/.tmp/B/f2e3abbc14ba4c25a89bd3bb426ad544 2024-11-18T20:12:23,631 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/.tmp/C/f6203fe303154cf8954019222620e9fd is 50, key is test_row_0/C:col10/1731960742125/Put/seqid=0 2024-11-18T20:12:23,645 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073741946_1122 (size=12301) 2024-11-18T20:12:23,666 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35963 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-18T20:12:23,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73. 2024-11-18T20:12:23,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73. as already flushing 2024-11-18T20:12:23,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73. 2024-11-18T20:12:23,667 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => 326884d1d40acf488e8bd50aea838c73, NAME => 'TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:12:23,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => 326884d1d40acf488e8bd50aea838c73, NAME => 'TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:12:23,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41277 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 326884d1d40acf488e8bd50aea838c73, NAME => 'TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 326884d1d40acf488e8bd50aea838c73, NAME => 'TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:12:23,819 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35963 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-18T20:12:23,819 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73. 2024-11-18T20:12:23,819 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73. as already flushing 2024-11-18T20:12:23,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73. 2024-11-18T20:12:23,820 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => 326884d1d40acf488e8bd50aea838c73, NAME => 'TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:12:23,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => 326884d1d40acf488e8bd50aea838c73, NAME => 'TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:12:23,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41277 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 326884d1d40acf488e8bd50aea838c73, NAME => 'TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 326884d1d40acf488e8bd50aea838c73, NAME => 'TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:12:23,943 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:23,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33046 deadline: 1731960803943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:23,944 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:23,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33070 deadline: 1731960803943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:23,945 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:23,945 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:23,945 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:23,945 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:23,945 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:12:23,945 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:12:23,956 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:23,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33072 deadline: 1731960803954, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:23,956 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:23,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33028 deadline: 1731960803954, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:23,957 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:23,957 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:23,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33056 deadline: 1731960803954, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:23,957 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:23,957 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:12:23,957 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:23,957 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:23,957 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:12:23,957 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:23,957 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:23,957 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:12:23,972 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35963 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-18T20:12:23,973 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73. 2024-11-18T20:12:23,973 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73. as already flushing 2024-11-18T20:12:23,973 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73. 2024-11-18T20:12:23,973 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => 326884d1d40acf488e8bd50aea838c73, NAME => 'TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:12:23,973 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => 326884d1d40acf488e8bd50aea838c73, NAME => 'TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:12:23,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41277 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 326884d1d40acf488e8bd50aea838c73, NAME => 'TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 326884d1d40acf488e8bd50aea838c73, NAME => 'TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:12:24,008 INFO [Thread-156 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-156. 2024-11-18T20:12:24,008 DEBUG [Thread-156 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-18T20:12:24,009 INFO [Thread-154 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-154. 2024-11-18T20:12:24,009 DEBUG [Thread-154 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-18T20:12:24,013 INFO [Thread-160 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-160. 2024-11-18T20:12:24,013 DEBUG [Thread-160 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-18T20:12:24,013 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:12:24,013 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:12:24,014 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:12:24,014 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:12:24,014 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T20:12:24,014 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T20:12:24,015 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:12:24,015 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:12:24,015 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T20:12:24,016 INFO [Thread-158 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-158. 2024-11-18T20:12:24,016 DEBUG [Thread-158 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-18T20:12:24,016 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:12:24,017 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:12:24,017 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T20:12:24,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-18T20:12:24,046 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=506 (bloomFilter=true), to=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/.tmp/C/f6203fe303154cf8954019222620e9fd 2024-11-18T20:12:24,052 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/.tmp/A/297625051e604ebc8357f0b03c23a862 as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/A/297625051e604ebc8357f0b03c23a862 2024-11-18T20:12:24,056 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/A/297625051e604ebc8357f0b03c23a862, entries=150, sequenceid=506, filesize=12.0 K 2024-11-18T20:12:24,057 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/.tmp/B/f2e3abbc14ba4c25a89bd3bb426ad544 as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/B/f2e3abbc14ba4c25a89bd3bb426ad544 2024-11-18T20:12:24,062 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/B/f2e3abbc14ba4c25a89bd3bb426ad544, entries=150, sequenceid=506, filesize=12.0 K 2024-11-18T20:12:24,063 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/.tmp/C/f6203fe303154cf8954019222620e9fd as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/C/f6203fe303154cf8954019222620e9fd 2024-11-18T20:12:24,067 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/C/f6203fe303154cf8954019222620e9fd, entries=150, sequenceid=506, filesize=12.0 K 2024-11-18T20:12:24,068 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for 326884d1d40acf488e8bd50aea838c73 in 1313ms, sequenceid=506, compaction requested=true 2024-11-18T20:12:24,068 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 326884d1d40acf488e8bd50aea838c73: 2024-11-18T20:12:24,069 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 326884d1d40acf488e8bd50aea838c73:A, priority=-2147483648, current under compaction store size is 1 2024-11-18T20:12:24,069 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T20:12:24,069 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:12:24,069 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 326884d1d40acf488e8bd50aea838c73:B, priority=-2147483648, current under compaction store size is 2 2024-11-18T20:12:24,069 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:12:24,069 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T20:12:24,069 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 326884d1d40acf488e8bd50aea838c73:C, priority=-2147483648, current under compaction store size is 3 2024-11-18T20:12:24,069 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T20:12:24,070 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38061 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T20:12:24,070 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38061 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T20:12:24,070 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HStore(1541): 326884d1d40acf488e8bd50aea838c73/A is initiating minor compaction (all files) 2024-11-18T20:12:24,070 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HStore(1541): 326884d1d40acf488e8bd50aea838c73/B is initiating minor compaction (all files) 2024-11-18T20:12:24,070 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 326884d1d40acf488e8bd50aea838c73/A in TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73. 2024-11-18T20:12:24,070 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 326884d1d40acf488e8bd50aea838c73/B in TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73. 2024-11-18T20:12:24,070 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/A/90f6af9cf92d4df293c47c65a5aa33e5, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/A/74369a8e70224776ae6b6ad11688cd1d, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/A/297625051e604ebc8357f0b03c23a862] into tmpdir=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/.tmp, totalSize=37.2 K 2024-11-18T20:12:24,070 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/B/952e1a9c24c7450cb645c7adeaf61807, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/B/5eb29e1f83e740efbdef5a43c9f0dd9c, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/B/f2e3abbc14ba4c25a89bd3bb426ad544] into tmpdir=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/.tmp, totalSize=37.2 K 2024-11-18T20:12:24,070 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.Compactor(225): Compacting 952e1a9c24c7450cb645c7adeaf61807, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=465, earliestPutTs=1731960740255 2024-11-18T20:12:24,070 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] compactions.Compactor(225): Compacting 90f6af9cf92d4df293c47c65a5aa33e5, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=465, earliestPutTs=1731960740255 2024-11-18T20:12:24,071 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.Compactor(225): Compacting 5eb29e1f83e740efbdef5a43c9f0dd9c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=490, earliestPutTs=1731960741465 2024-11-18T20:12:24,071 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] compactions.Compactor(225): Compacting 74369a8e70224776ae6b6ad11688cd1d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=490, earliestPutTs=1731960741465 2024-11-18T20:12:24,071 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.Compactor(225): Compacting f2e3abbc14ba4c25a89bd3bb426ad544, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=506, earliestPutTs=1731960742125 2024-11-18T20:12:24,071 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] compactions.Compactor(225): Compacting 297625051e604ebc8357f0b03c23a862, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=506, earliestPutTs=1731960742125 2024-11-18T20:12:24,080 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 326884d1d40acf488e8bd50aea838c73#B#compaction#110 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:12:24,081 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/.tmp/B/ca27638f2c4345208df0f1e36e41490a is 50, key is test_row_0/B:col10/1731960742125/Put/seqid=0 2024-11-18T20:12:24,081 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 326884d1d40acf488e8bd50aea838c73#A#compaction#111 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:12:24,081 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/.tmp/A/3ab08836ff0f438fa80f0ddba9d6341f is 50, key is test_row_0/A:col10/1731960742125/Put/seqid=0 2024-11-18T20:12:24,087 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073741947_1123 (size=13561) 2024-11-18T20:12:24,094 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073741948_1124 (size=13561) 2024-11-18T20:12:24,126 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35963 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-18T20:12:24,126 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73. 2024-11-18T20:12:24,126 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2902): Flushing 326884d1d40acf488e8bd50aea838c73 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-18T20:12:24,132 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/.tmp/A/39c0fd20d09247e18aa3a0854dca1d0b is 50, key is test_row_0/A:col10/1731960742801/Put/seqid=0 2024-11-18T20:12:24,136 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073741949_1125 (size=12301) 2024-11-18T20:12:24,137 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=529 (bloomFilter=true), to=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/.tmp/A/39c0fd20d09247e18aa3a0854dca1d0b 2024-11-18T20:12:24,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/.tmp/B/e8bb5f4af1ff4732914f997cb80879ad is 50, key is test_row_0/B:col10/1731960742801/Put/seqid=0 2024-11-18T20:12:24,159 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073741950_1126 (size=12301) 2024-11-18T20:12:24,493 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/.tmp/B/ca27638f2c4345208df0f1e36e41490a as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/B/ca27638f2c4345208df0f1e36e41490a 2024-11-18T20:12:24,499 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 326884d1d40acf488e8bd50aea838c73/B of 326884d1d40acf488e8bd50aea838c73 into ca27638f2c4345208df0f1e36e41490a(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:12:24,499 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 326884d1d40acf488e8bd50aea838c73: 2024-11-18T20:12:24,499 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., storeName=326884d1d40acf488e8bd50aea838c73/B, priority=13, startTime=1731960744069; duration=0sec 2024-11-18T20:12:24,500 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T20:12:24,500 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 326884d1d40acf488e8bd50aea838c73:B 2024-11-18T20:12:24,500 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/.tmp/A/3ab08836ff0f438fa80f0ddba9d6341f as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/A/3ab08836ff0f438fa80f0ddba9d6341f 2024-11-18T20:12:24,500 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T20:12:24,500 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38027 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T20:12:24,501 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HStore(1541): 326884d1d40acf488e8bd50aea838c73/C is initiating minor compaction (all files) 2024-11-18T20:12:24,501 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 326884d1d40acf488e8bd50aea838c73/C in TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73. 2024-11-18T20:12:24,501 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/C/79a7f500da114675bd904acea264b4fe, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/C/6a4ad8aeebf541e3a78afbb395b5e0e3, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/C/f6203fe303154cf8954019222620e9fd] into tmpdir=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/.tmp, totalSize=37.1 K 2024-11-18T20:12:24,501 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.Compactor(225): Compacting 79a7f500da114675bd904acea264b4fe, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=465, earliestPutTs=1731960740255 2024-11-18T20:12:24,502 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.Compactor(225): Compacting 6a4ad8aeebf541e3a78afbb395b5e0e3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=490, earliestPutTs=1731960741465 2024-11-18T20:12:24,502 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.Compactor(225): Compacting f6203fe303154cf8954019222620e9fd, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=506, earliestPutTs=1731960742125 2024-11-18T20:12:24,505 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 326884d1d40acf488e8bd50aea838c73/A of 326884d1d40acf488e8bd50aea838c73 into 3ab08836ff0f438fa80f0ddba9d6341f(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:12:24,505 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 326884d1d40acf488e8bd50aea838c73: 2024-11-18T20:12:24,506 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., storeName=326884d1d40acf488e8bd50aea838c73/A, priority=13, startTime=1731960744068; duration=0sec 2024-11-18T20:12:24,506 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:12:24,506 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 326884d1d40acf488e8bd50aea838c73:A 2024-11-18T20:12:24,513 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 326884d1d40acf488e8bd50aea838c73#C#compaction#114 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:12:24,514 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/.tmp/C/df42d6bcce78435ebdc53fa5de962093 is 50, key is test_row_0/C:col10/1731960742125/Put/seqid=0 2024-11-18T20:12:24,517 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073741951_1127 (size=13527) 2024-11-18T20:12:24,560 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=529 (bloomFilter=true), to=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/.tmp/B/e8bb5f4af1ff4732914f997cb80879ad 2024-11-18T20:12:24,569 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/.tmp/C/e7cb4284c79040b9baadc4ae6233ceed is 50, key is test_row_0/C:col10/1731960742801/Put/seqid=0 2024-11-18T20:12:24,574 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073741952_1128 (size=12301) 2024-11-18T20:12:24,933 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/.tmp/C/df42d6bcce78435ebdc53fa5de962093 as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/C/df42d6bcce78435ebdc53fa5de962093 2024-11-18T20:12:24,941 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 326884d1d40acf488e8bd50aea838c73/C of 326884d1d40acf488e8bd50aea838c73 into df42d6bcce78435ebdc53fa5de962093(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:12:24,941 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 326884d1d40acf488e8bd50aea838c73: 2024-11-18T20:12:24,941 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., storeName=326884d1d40acf488e8bd50aea838c73/C, priority=13, startTime=1731960744069; duration=0sec 2024-11-18T20:12:24,941 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:12:24,941 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 326884d1d40acf488e8bd50aea838c73:C 2024-11-18T20:12:24,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(8855): Flush requested on 326884d1d40acf488e8bd50aea838c73 2024-11-18T20:12:24,955 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73. as already flushing 2024-11-18T20:12:24,956 INFO [Thread-145 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-145. 2024-11-18T20:12:24,956 INFO [Thread-149 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-149. 2024-11-18T20:12:24,956 DEBUG [Thread-145 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-18T20:12:24,956 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:12:24,956 DEBUG [Thread-149 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-18T20:12:24,956 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:12:24,956 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:12:24,956 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T20:12:24,956 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:12:24,956 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T20:12:24,965 INFO [Thread-151 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-151. 2024-11-18T20:12:24,965 DEBUG [Thread-151 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-18T20:12:24,965 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:12:24,965 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:12:24,965 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T20:12:24,976 INFO [Thread-143 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-143. 2024-11-18T20:12:24,976 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=529 (bloomFilter=true), to=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/.tmp/C/e7cb4284c79040b9baadc4ae6233ceed 2024-11-18T20:12:24,976 DEBUG [Thread-143 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-18T20:12:24,976 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:12:24,976 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:12:24,976 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T20:12:24,976 INFO [Thread-147 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-147. 2024-11-18T20:12:24,976 DEBUG [Thread-147 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-18T20:12:24,976 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:12:24,977 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:12:24,977 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T20:12:24,984 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/.tmp/A/39c0fd20d09247e18aa3a0854dca1d0b as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/A/39c0fd20d09247e18aa3a0854dca1d0b 2024-11-18T20:12:24,990 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/A/39c0fd20d09247e18aa3a0854dca1d0b, entries=150, sequenceid=529, filesize=12.0 K 2024-11-18T20:12:24,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/.tmp/B/e8bb5f4af1ff4732914f997cb80879ad as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/B/e8bb5f4af1ff4732914f997cb80879ad 2024-11-18T20:12:24,998 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/B/e8bb5f4af1ff4732914f997cb80879ad, entries=150, sequenceid=529, filesize=12.0 K 2024-11-18T20:12:24,999 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/.tmp/C/e7cb4284c79040b9baadc4ae6233ceed as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/C/e7cb4284c79040b9baadc4ae6233ceed 2024-11-18T20:12:25,004 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/C/e7cb4284c79040b9baadc4ae6233ceed, entries=150, sequenceid=529, filesize=12.0 K 2024-11-18T20:12:25,005 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=33.54 KB/34350 for 326884d1d40acf488e8bd50aea838c73 in 879ms, sequenceid=529, compaction requested=false 2024-11-18T20:12:25,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2603): Flush status journal for 326884d1d40acf488e8bd50aea838c73: 2024-11-18T20:12:25,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73. 2024-11-18T20:12:25,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=26 2024-11-18T20:12:25,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41277 {}] master.HMaster(4169): Remote procedure done, pid=26 2024-11-18T20:12:25,009 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=26, resume processing ppid=25 2024-11-18T20:12:25,009 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=26, ppid=25, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1100 sec 2024-11-18T20:12:25,011 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=25, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees in 2.1150 sec 2024-11-18T20:12:25,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-18T20:12:25,043 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-18T20:12:25,043 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-18T20:12:25,043 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 90 2024-11-18T20:12:25,043 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 81 2024-11-18T20:12:25,043 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 94 2024-11-18T20:12:25,043 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 72 2024-11-18T20:12:25,043 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 83 2024-11-18T20:12:25,044 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-18T20:12:25,044 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3284 2024-11-18T20:12:25,044 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3156 2024-11-18T20:12:25,044 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-18T20:12:25,044 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1515 2024-11-18T20:12:25,044 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4539 rows 2024-11-18T20:12:25,044 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1517 2024-11-18T20:12:25,044 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4548 rows 2024-11-18T20:12:25,044 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-18T20:12:25,044 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMixedAtomicity(AcidGuaranteesTestBase.java:117) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-18T20:12:25,044 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:12:25,044 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:12:25,044 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T20:12:25,048 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41277 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-18T20:12:25,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41277 {}] procedure2.ProcedureExecutor(1139): Stored pid=27, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-18T20:12:25,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-18T20:12:25,057 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731960745056"}]},"ts":"1731960745056"} 2024-11-18T20:12:25,059 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-18T20:12:25,059 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-18T20:12:25,061 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=28, ppid=27, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-18T20:12:25,066 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=326884d1d40acf488e8bd50aea838c73, UNASSIGN}] 2024-11-18T20:12:25,067 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=326884d1d40acf488e8bd50aea838c73, UNASSIGN 2024-11-18T20:12:25,068 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=326884d1d40acf488e8bd50aea838c73, regionState=CLOSING, regionLocation=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:25,070 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=326884d1d40acf488e8bd50aea838c73, UNASSIGN because future has completed 2024-11-18T20:12:25,070 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-18T20:12:25,071 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=30, ppid=29, state=RUNNABLE, hasLock=false; CloseRegionProcedure 326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223}] 2024-11-18T20:12:25,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-18T20:12:25,229 INFO [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] handler.UnassignRegionHandler(122): Close 326884d1d40acf488e8bd50aea838c73 2024-11-18T20:12:25,229 DEBUG [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-18T20:12:25,230 DEBUG [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1722): Closing 326884d1d40acf488e8bd50aea838c73, disabling compactions & flushes 2024-11-18T20:12:25,230 INFO [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73. 2024-11-18T20:12:25,230 DEBUG [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73. 2024-11-18T20:12:25,231 DEBUG [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73. after waiting 0 ms 2024-11-18T20:12:25,231 DEBUG [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73. 2024-11-18T20:12:25,231 INFO [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(2902): Flushing 326884d1d40acf488e8bd50aea838c73 3/3 column families, dataSize=33.54 KB heapSize=88.64 KB 2024-11-18T20:12:25,238 DEBUG [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/.tmp/A/2a1ada93f2ac49588ed65010244db0ba is 50, key is test_row_0/A:col10/1731960744973/Put/seqid=0 2024-11-18T20:12:25,242 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073741953_1129 (size=9857) 2024-11-18T20:12:25,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-18T20:12:25,644 INFO [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=540 (bloomFilter=true), to=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/.tmp/A/2a1ada93f2ac49588ed65010244db0ba 2024-11-18T20:12:25,657 DEBUG [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/.tmp/B/69c9420987944b9a807fde6a13b0bb76 is 50, key is test_row_0/B:col10/1731960744973/Put/seqid=0 2024-11-18T20:12:25,660 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073741954_1130 (size=9857) 2024-11-18T20:12:25,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-18T20:12:26,061 INFO [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=540 (bloomFilter=true), to=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/.tmp/B/69c9420987944b9a807fde6a13b0bb76 2024-11-18T20:12:26,070 DEBUG [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/.tmp/C/0948f0f9726a45819bfcecaec4921cbd is 50, key is test_row_0/C:col10/1731960744973/Put/seqid=0 2024-11-18T20:12:26,074 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073741955_1131 (size=9857) 2024-11-18T20:12:26,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-18T20:12:26,475 INFO [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=540 (bloomFilter=true), to=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/.tmp/C/0948f0f9726a45819bfcecaec4921cbd 2024-11-18T20:12:26,481 DEBUG [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/.tmp/A/2a1ada93f2ac49588ed65010244db0ba as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/A/2a1ada93f2ac49588ed65010244db0ba 2024-11-18T20:12:26,486 INFO [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/A/2a1ada93f2ac49588ed65010244db0ba, entries=100, sequenceid=540, filesize=9.6 K 2024-11-18T20:12:26,488 DEBUG [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/.tmp/B/69c9420987944b9a807fde6a13b0bb76 as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/B/69c9420987944b9a807fde6a13b0bb76 2024-11-18T20:12:26,494 INFO [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/B/69c9420987944b9a807fde6a13b0bb76, entries=100, sequenceid=540, filesize=9.6 K 2024-11-18T20:12:26,495 DEBUG [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/.tmp/C/0948f0f9726a45819bfcecaec4921cbd as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/C/0948f0f9726a45819bfcecaec4921cbd 2024-11-18T20:12:26,500 INFO [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/C/0948f0f9726a45819bfcecaec4921cbd, entries=100, sequenceid=540, filesize=9.6 K 2024-11-18T20:12:26,501 INFO [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for 326884d1d40acf488e8bd50aea838c73 in 1270ms, sequenceid=540, compaction requested=true 2024-11-18T20:12:26,502 DEBUG [StoreCloser-TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/A/124683c738434fc8a0d3b04126c95f32, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/A/fc0a6baebd6d4bf9b155a74d4878d413, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/A/a5d33b2a90f449abb085753542d95840, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/A/cc1bcccd02b64d9ea6fcbc96155bafdf, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/A/058f1958519b4d97b9c8fdcfe4ace58e, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/A/fbeb1d89f58c4ef59f662a14bd2f04c1, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/A/9de89760a89a4c3ab5ce08f397185ffb, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/A/978e5753f01b4e7eb96e9bc30d36ac1c, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/A/a4cb193dc4094156908221894e02b71e, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/A/7d324f29469a4152a7dff5a74672c4ea, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/A/47e69739427c498aa338823af7293720, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/A/dd8f3a6186344a34b25bf0e72add49a4, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/A/33812c745aee4ce8bddf200a129b9f9b, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/A/62c2388048994dd2a0162d466fcc1c10, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/A/12ef51908c6e4012b11635f0a3bd529b, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/A/21ad72f14a1d41348947ef07c8c74675, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/A/e0ca3c9891654ec5a4ffdb50923e34ca, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/A/a4e5a680ef9248849f0b481698f0751b, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/A/1bfc3b275e9f44fbb794e1e5efef9013, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/A/0c882164965a42689763478e96370b67, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/A/0c4edfba4c814e82ae3cdd6576d9986f, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/A/eed6ca7c8b1e4f8fb698b82541d43457, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/A/54c56f57a25247189b02168d4a6babd4, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/A/7e650756d1ee489cb67cc1801bee88da, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/A/036d06f8299e421ca2d59e975ba5b121, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/A/8cb3994e0f2c45538a70a12c7f7d5007, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/A/8d505235538c4e00a2e80fc5deba2fb8, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/A/4adf08c5d45f42d895d681c86b015852, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/A/8727132e5ba546b7b5f9ad74aa3928f8, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/A/822d31fcded2479f9e073562bb3da745, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/A/7030952e566349e99f3949ae7842129c, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/A/403d977ac56a4e4aba816ed73dcc0ea1, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/A/4b796b62349740489ebc01ceb833f45e, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/A/db998326449549ef9212fd525c6b6217, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/A/90f6af9cf92d4df293c47c65a5aa33e5, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/A/74369a8e70224776ae6b6ad11688cd1d, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/A/297625051e604ebc8357f0b03c23a862] to archive 2024-11-18T20:12:26,505 DEBUG [StoreCloser-TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-18T20:12:26,512 DEBUG [StoreCloser-TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/A/124683c738434fc8a0d3b04126c95f32 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/A/124683c738434fc8a0d3b04126c95f32 2024-11-18T20:12:26,516 DEBUG [StoreCloser-TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/A/fc0a6baebd6d4bf9b155a74d4878d413 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/A/fc0a6baebd6d4bf9b155a74d4878d413 2024-11-18T20:12:26,522 DEBUG [StoreCloser-TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/A/a5d33b2a90f449abb085753542d95840 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/A/a5d33b2a90f449abb085753542d95840 2024-11-18T20:12:26,524 DEBUG [StoreCloser-TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/A/cc1bcccd02b64d9ea6fcbc96155bafdf to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/A/cc1bcccd02b64d9ea6fcbc96155bafdf 2024-11-18T20:12:26,526 DEBUG [StoreCloser-TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/A/058f1958519b4d97b9c8fdcfe4ace58e to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/A/058f1958519b4d97b9c8fdcfe4ace58e 2024-11-18T20:12:26,528 DEBUG [StoreCloser-TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/A/fbeb1d89f58c4ef59f662a14bd2f04c1 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/A/fbeb1d89f58c4ef59f662a14bd2f04c1 2024-11-18T20:12:26,530 DEBUG [StoreCloser-TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/A/9de89760a89a4c3ab5ce08f397185ffb to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/A/9de89760a89a4c3ab5ce08f397185ffb 2024-11-18T20:12:26,532 DEBUG [StoreCloser-TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/A/978e5753f01b4e7eb96e9bc30d36ac1c to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/A/978e5753f01b4e7eb96e9bc30d36ac1c 2024-11-18T20:12:26,533 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-18T20:12:26,535 DEBUG [StoreCloser-TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/A/a4cb193dc4094156908221894e02b71e to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/A/a4cb193dc4094156908221894e02b71e 2024-11-18T20:12:26,537 DEBUG [StoreCloser-TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/A/7d324f29469a4152a7dff5a74672c4ea to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/A/7d324f29469a4152a7dff5a74672c4ea 2024-11-18T20:12:26,539 DEBUG [StoreCloser-TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/A/47e69739427c498aa338823af7293720 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/A/47e69739427c498aa338823af7293720 2024-11-18T20:12:26,542 DEBUG [StoreCloser-TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/A/dd8f3a6186344a34b25bf0e72add49a4 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/A/dd8f3a6186344a34b25bf0e72add49a4 2024-11-18T20:12:26,545 DEBUG [StoreCloser-TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/A/33812c745aee4ce8bddf200a129b9f9b to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/A/33812c745aee4ce8bddf200a129b9f9b 2024-11-18T20:12:26,547 DEBUG [StoreCloser-TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/A/62c2388048994dd2a0162d466fcc1c10 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/A/62c2388048994dd2a0162d466fcc1c10 2024-11-18T20:12:26,550 DEBUG [StoreCloser-TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/A/12ef51908c6e4012b11635f0a3bd529b to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/A/12ef51908c6e4012b11635f0a3bd529b 2024-11-18T20:12:26,552 DEBUG [StoreCloser-TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/A/21ad72f14a1d41348947ef07c8c74675 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/A/21ad72f14a1d41348947ef07c8c74675 2024-11-18T20:12:26,553 DEBUG [StoreCloser-TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/A/e0ca3c9891654ec5a4ffdb50923e34ca to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/A/e0ca3c9891654ec5a4ffdb50923e34ca 2024-11-18T20:12:26,555 DEBUG [StoreCloser-TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/A/a4e5a680ef9248849f0b481698f0751b to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/A/a4e5a680ef9248849f0b481698f0751b 2024-11-18T20:12:26,556 DEBUG [StoreCloser-TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/A/1bfc3b275e9f44fbb794e1e5efef9013 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/A/1bfc3b275e9f44fbb794e1e5efef9013 2024-11-18T20:12:26,557 DEBUG [StoreCloser-TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/A/0c882164965a42689763478e96370b67 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/A/0c882164965a42689763478e96370b67 2024-11-18T20:12:26,559 DEBUG [StoreCloser-TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/A/0c4edfba4c814e82ae3cdd6576d9986f to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/A/0c4edfba4c814e82ae3cdd6576d9986f 2024-11-18T20:12:26,560 DEBUG [StoreCloser-TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/A/eed6ca7c8b1e4f8fb698b82541d43457 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/A/eed6ca7c8b1e4f8fb698b82541d43457 2024-11-18T20:12:26,561 DEBUG [StoreCloser-TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/A/54c56f57a25247189b02168d4a6babd4 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/A/54c56f57a25247189b02168d4a6babd4 2024-11-18T20:12:26,563 DEBUG [StoreCloser-TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/A/7e650756d1ee489cb67cc1801bee88da to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/A/7e650756d1ee489cb67cc1801bee88da 2024-11-18T20:12:26,564 DEBUG [StoreCloser-TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/A/036d06f8299e421ca2d59e975ba5b121 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/A/036d06f8299e421ca2d59e975ba5b121 2024-11-18T20:12:26,565 DEBUG [StoreCloser-TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/A/8cb3994e0f2c45538a70a12c7f7d5007 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/A/8cb3994e0f2c45538a70a12c7f7d5007 2024-11-18T20:12:26,567 DEBUG [StoreCloser-TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/A/8d505235538c4e00a2e80fc5deba2fb8 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/A/8d505235538c4e00a2e80fc5deba2fb8 2024-11-18T20:12:26,568 DEBUG [StoreCloser-TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/A/4adf08c5d45f42d895d681c86b015852 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/A/4adf08c5d45f42d895d681c86b015852 2024-11-18T20:12:26,569 DEBUG [StoreCloser-TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/A/8727132e5ba546b7b5f9ad74aa3928f8 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/A/8727132e5ba546b7b5f9ad74aa3928f8 2024-11-18T20:12:26,570 DEBUG [StoreCloser-TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/A/822d31fcded2479f9e073562bb3da745 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/A/822d31fcded2479f9e073562bb3da745 2024-11-18T20:12:26,571 DEBUG [StoreCloser-TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/A/7030952e566349e99f3949ae7842129c to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/A/7030952e566349e99f3949ae7842129c 2024-11-18T20:12:26,572 DEBUG [StoreCloser-TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/A/403d977ac56a4e4aba816ed73dcc0ea1 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/A/403d977ac56a4e4aba816ed73dcc0ea1 2024-11-18T20:12:26,574 DEBUG [StoreCloser-TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/A/4b796b62349740489ebc01ceb833f45e to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/A/4b796b62349740489ebc01ceb833f45e 2024-11-18T20:12:26,575 DEBUG [StoreCloser-TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/A/db998326449549ef9212fd525c6b6217 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/A/db998326449549ef9212fd525c6b6217 2024-11-18T20:12:26,576 DEBUG [StoreCloser-TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/A/90f6af9cf92d4df293c47c65a5aa33e5 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/A/90f6af9cf92d4df293c47c65a5aa33e5 2024-11-18T20:12:26,577 DEBUG [StoreCloser-TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/A/74369a8e70224776ae6b6ad11688cd1d to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/A/74369a8e70224776ae6b6ad11688cd1d 2024-11-18T20:12:26,578 DEBUG [StoreCloser-TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/A/297625051e604ebc8357f0b03c23a862 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/A/297625051e604ebc8357f0b03c23a862 2024-11-18T20:12:26,591 DEBUG [StoreCloser-TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/B/bbffcc63619d435293ea9b31e14d1e5b, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/B/cd9e75d138ad4dcc8e1d330154ce4518, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/B/44342c347d4d4d61acc1558c507ecacc, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/B/bab568e8a5074950986327999f0c32ae, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/B/5e38cff1b0104ee7b10b42cd81c8b66f, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/B/adfdd57b05bc4ce09a93f307c548e341, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/B/9d617b6b7db14c03a90592af7b61f17f, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/B/ea3e4562667b4f709a4167a7ed94421d, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/B/8f1b446ff4c740d8b5bed39e5ae5d758, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/B/67d27c3ba91c44faa70cc786fcac0d7a, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/B/f0a5d5a160bd4c8badea0af09518bde9, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/B/a77f383b179545ec8b284f279bdeb156, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/B/cfc19337f7d1465584f715a6e740d167, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/B/9763ff43338141c2a7be872384367835, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/B/83d49644bc2840ca893579a7a3495c85, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/B/1671a674abe646418de4581d5ac37ac4, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/B/535b3a85d3684883bcf403ce3cc452de, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/B/e74fd6ec3a534ac0afa00c2c9eb2e03b, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/B/e49e53b8d7ca4f81b7e82677ea17c404, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/B/bc34dc5c7a4440bbae1fdc8cde503796, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/B/7fbde023e43e44528f4bd740f951bfe4, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/B/d602ca52155d4bbba2fa42148d2181b2, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/B/dfeeefecd0094ceabb9c4d294273533f, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/B/d5a9d670ca964a7d88a1c8cd11d2df8e, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/B/35279005022e483f8c8b7d07426de443, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/B/3db78eb034354c9abe17c0901ce2e61a, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/B/660b3de1c9994802b0f9769e933b7fa0, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/B/45b0482610194430993171686f5e8582, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/B/be10d891706146c6a4715cb1cac38f71, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/B/b4880fc6b7a14c0c9cdf976ce0fd8595, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/B/d0d1139f7ebe4351acba24449ed94fc4, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/B/3f239ac7e53a434fbfc59a140b149e5f, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/B/0fb361e5aff54623bc5860937e9c8f58, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/B/952e1a9c24c7450cb645c7adeaf61807, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/B/a5db33483c20489da7afa0ff39f1ffe3, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/B/5eb29e1f83e740efbdef5a43c9f0dd9c, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/B/f2e3abbc14ba4c25a89bd3bb426ad544] to archive 2024-11-18T20:12:26,592 DEBUG [StoreCloser-TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-18T20:12:26,593 DEBUG [StoreCloser-TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/B/bbffcc63619d435293ea9b31e14d1e5b to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/B/bbffcc63619d435293ea9b31e14d1e5b 2024-11-18T20:12:26,594 DEBUG [StoreCloser-TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/B/cd9e75d138ad4dcc8e1d330154ce4518 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/B/cd9e75d138ad4dcc8e1d330154ce4518 2024-11-18T20:12:26,595 DEBUG [StoreCloser-TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/B/44342c347d4d4d61acc1558c507ecacc to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/B/44342c347d4d4d61acc1558c507ecacc 2024-11-18T20:12:26,596 DEBUG [StoreCloser-TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/B/bab568e8a5074950986327999f0c32ae to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/B/bab568e8a5074950986327999f0c32ae 2024-11-18T20:12:26,597 DEBUG [StoreCloser-TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/B/5e38cff1b0104ee7b10b42cd81c8b66f to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/B/5e38cff1b0104ee7b10b42cd81c8b66f 2024-11-18T20:12:26,599 DEBUG [StoreCloser-TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/B/adfdd57b05bc4ce09a93f307c548e341 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/B/adfdd57b05bc4ce09a93f307c548e341 2024-11-18T20:12:26,600 DEBUG [StoreCloser-TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/B/9d617b6b7db14c03a90592af7b61f17f to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/B/9d617b6b7db14c03a90592af7b61f17f 2024-11-18T20:12:26,601 DEBUG [StoreCloser-TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/B/ea3e4562667b4f709a4167a7ed94421d to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/B/ea3e4562667b4f709a4167a7ed94421d 2024-11-18T20:12:26,603 DEBUG [StoreCloser-TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/B/8f1b446ff4c740d8b5bed39e5ae5d758 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/B/8f1b446ff4c740d8b5bed39e5ae5d758 2024-11-18T20:12:26,604 DEBUG [StoreCloser-TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/B/67d27c3ba91c44faa70cc786fcac0d7a to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/B/67d27c3ba91c44faa70cc786fcac0d7a 2024-11-18T20:12:26,605 DEBUG [StoreCloser-TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/B/f0a5d5a160bd4c8badea0af09518bde9 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/B/f0a5d5a160bd4c8badea0af09518bde9 2024-11-18T20:12:26,606 DEBUG [StoreCloser-TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/B/a77f383b179545ec8b284f279bdeb156 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/B/a77f383b179545ec8b284f279bdeb156 2024-11-18T20:12:26,607 DEBUG [StoreCloser-TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/B/cfc19337f7d1465584f715a6e740d167 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/B/cfc19337f7d1465584f715a6e740d167 2024-11-18T20:12:26,609 DEBUG [StoreCloser-TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/B/9763ff43338141c2a7be872384367835 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/B/9763ff43338141c2a7be872384367835 2024-11-18T20:12:26,610 DEBUG [StoreCloser-TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/B/83d49644bc2840ca893579a7a3495c85 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/B/83d49644bc2840ca893579a7a3495c85 2024-11-18T20:12:26,611 DEBUG [StoreCloser-TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/B/1671a674abe646418de4581d5ac37ac4 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/B/1671a674abe646418de4581d5ac37ac4 2024-11-18T20:12:26,611 DEBUG [StoreCloser-TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/B/535b3a85d3684883bcf403ce3cc452de to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/B/535b3a85d3684883bcf403ce3cc452de 2024-11-18T20:12:26,613 DEBUG [StoreCloser-TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/B/e74fd6ec3a534ac0afa00c2c9eb2e03b to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/B/e74fd6ec3a534ac0afa00c2c9eb2e03b 2024-11-18T20:12:26,614 DEBUG [StoreCloser-TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/B/e49e53b8d7ca4f81b7e82677ea17c404 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/B/e49e53b8d7ca4f81b7e82677ea17c404 2024-11-18T20:12:26,615 DEBUG [StoreCloser-TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/B/bc34dc5c7a4440bbae1fdc8cde503796 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/B/bc34dc5c7a4440bbae1fdc8cde503796 2024-11-18T20:12:26,615 DEBUG [StoreCloser-TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/B/7fbde023e43e44528f4bd740f951bfe4 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/B/7fbde023e43e44528f4bd740f951bfe4 2024-11-18T20:12:26,616 DEBUG [StoreCloser-TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/B/d602ca52155d4bbba2fa42148d2181b2 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/B/d602ca52155d4bbba2fa42148d2181b2 2024-11-18T20:12:26,617 DEBUG [StoreCloser-TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/B/dfeeefecd0094ceabb9c4d294273533f to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/B/dfeeefecd0094ceabb9c4d294273533f 2024-11-18T20:12:26,618 DEBUG [StoreCloser-TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/B/d5a9d670ca964a7d88a1c8cd11d2df8e to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/B/d5a9d670ca964a7d88a1c8cd11d2df8e 2024-11-18T20:12:26,619 DEBUG [StoreCloser-TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/B/35279005022e483f8c8b7d07426de443 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/B/35279005022e483f8c8b7d07426de443 2024-11-18T20:12:26,620 DEBUG [StoreCloser-TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/B/3db78eb034354c9abe17c0901ce2e61a to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/B/3db78eb034354c9abe17c0901ce2e61a 2024-11-18T20:12:26,621 DEBUG [StoreCloser-TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/B/660b3de1c9994802b0f9769e933b7fa0 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/B/660b3de1c9994802b0f9769e933b7fa0 2024-11-18T20:12:26,622 DEBUG [StoreCloser-TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/B/45b0482610194430993171686f5e8582 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/B/45b0482610194430993171686f5e8582 2024-11-18T20:12:26,623 DEBUG [StoreCloser-TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/B/be10d891706146c6a4715cb1cac38f71 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/B/be10d891706146c6a4715cb1cac38f71 2024-11-18T20:12:26,624 DEBUG [StoreCloser-TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/B/b4880fc6b7a14c0c9cdf976ce0fd8595 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/B/b4880fc6b7a14c0c9cdf976ce0fd8595 2024-11-18T20:12:26,625 DEBUG [StoreCloser-TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/B/d0d1139f7ebe4351acba24449ed94fc4 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/B/d0d1139f7ebe4351acba24449ed94fc4 2024-11-18T20:12:26,626 DEBUG [StoreCloser-TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/B/3f239ac7e53a434fbfc59a140b149e5f to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/B/3f239ac7e53a434fbfc59a140b149e5f 2024-11-18T20:12:26,627 DEBUG [StoreCloser-TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/B/0fb361e5aff54623bc5860937e9c8f58 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/B/0fb361e5aff54623bc5860937e9c8f58 2024-11-18T20:12:26,629 DEBUG [StoreCloser-TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/B/952e1a9c24c7450cb645c7adeaf61807 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/B/952e1a9c24c7450cb645c7adeaf61807 2024-11-18T20:12:26,630 DEBUG [StoreCloser-TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/B/a5db33483c20489da7afa0ff39f1ffe3 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/B/a5db33483c20489da7afa0ff39f1ffe3 2024-11-18T20:12:26,631 DEBUG [StoreCloser-TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/B/5eb29e1f83e740efbdef5a43c9f0dd9c to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/B/5eb29e1f83e740efbdef5a43c9f0dd9c 2024-11-18T20:12:26,632 DEBUG [StoreCloser-TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/B/f2e3abbc14ba4c25a89bd3bb426ad544 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/B/f2e3abbc14ba4c25a89bd3bb426ad544 2024-11-18T20:12:26,634 DEBUG [StoreCloser-TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/C/04b4ce1131474dbab3d596c8d7fff1fb, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/C/df1f4b436c404217ace3b57d8b57be04, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/C/5bba0283845d4a62835670bf54969f5d, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/C/e2c2956f4098444983416b57cd15b95d, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/C/0eb22e3ba7fe404ab6a8e4b81c4d7ffe, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/C/afebc169e18c46c9919d3ee3b05ec557, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/C/5a28bdaea97745d6a8cecbe63f246849, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/C/8e03a13d858b4feda1348a2ae904fc6d, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/C/575a85dc630d4f7fbe04a6f8e64b7874, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/C/9f393cdd80ce414790fc9040d5f8fbcc, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/C/74f1d8e339d342a9ade550c080960874, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/C/6ee05491a9b94a5c86dee6e8fd3fa393, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/C/427bbd64ca6947c9b52afa0d85685f9a, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/C/7c807d6029364cdf8693b06bc4cf0604, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/C/afe362d8843742188e5c63ff3c9ce7d8, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/C/bd5835e1e6ac4244b2e746fac8937694, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/C/bc865bcfd2744007984ced955360e6f3, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/C/71f170f01f364dac9c12b546f53e20fa, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/C/ff309597fdc6400bb092706f52627c2e, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/C/591d46c5d1434763a6c7c077da1dce7a, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/C/56d32afce2e246c6baac00f288e84104, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/C/d4e8b147be684d3099d64db920d4e498, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/C/0fcb7ac7831f4238a2d626df07f5cf08, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/C/f472bbab37374b50b653dca80bbc51ac, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/C/be76e94061504f36b664767f51e08c86, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/C/1a4e8da4738b4b9bb9fc00e6ff0346f4, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/C/30143802614f4f2d943e803bb5ada559, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/C/d318e0d36ef04bbfa81b4a9cec0bed57, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/C/4d511892eb394f2e87f47ae4755c8405, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/C/0bf801180970467380d807e7f4462a63, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/C/ae65c7a9fa38497ca2777d173a4a27a4, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/C/234cc462460544368a9fdfd2b1c17cfc, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/C/79a7f500da114675bd904acea264b4fe, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/C/14c7cc34aa9f4feb9d610a1b8c652b4c, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/C/6a4ad8aeebf541e3a78afbb395b5e0e3, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/C/f6203fe303154cf8954019222620e9fd] to archive 2024-11-18T20:12:26,635 DEBUG [StoreCloser-TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-18T20:12:26,636 DEBUG [StoreCloser-TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/C/04b4ce1131474dbab3d596c8d7fff1fb to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/C/04b4ce1131474dbab3d596c8d7fff1fb 2024-11-18T20:12:26,637 DEBUG [StoreCloser-TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/C/df1f4b436c404217ace3b57d8b57be04 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/C/df1f4b436c404217ace3b57d8b57be04 2024-11-18T20:12:26,638 DEBUG [StoreCloser-TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/C/5bba0283845d4a62835670bf54969f5d to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/C/5bba0283845d4a62835670bf54969f5d 2024-11-18T20:12:26,639 DEBUG [StoreCloser-TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/C/e2c2956f4098444983416b57cd15b95d to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/C/e2c2956f4098444983416b57cd15b95d 2024-11-18T20:12:26,640 DEBUG [StoreCloser-TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/C/0eb22e3ba7fe404ab6a8e4b81c4d7ffe to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/C/0eb22e3ba7fe404ab6a8e4b81c4d7ffe 2024-11-18T20:12:26,641 DEBUG [StoreCloser-TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/C/afebc169e18c46c9919d3ee3b05ec557 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/C/afebc169e18c46c9919d3ee3b05ec557 2024-11-18T20:12:26,642 DEBUG [StoreCloser-TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/C/5a28bdaea97745d6a8cecbe63f246849 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/C/5a28bdaea97745d6a8cecbe63f246849 2024-11-18T20:12:26,643 DEBUG [StoreCloser-TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/C/8e03a13d858b4feda1348a2ae904fc6d to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/C/8e03a13d858b4feda1348a2ae904fc6d 2024-11-18T20:12:26,644 DEBUG [StoreCloser-TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/C/575a85dc630d4f7fbe04a6f8e64b7874 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/C/575a85dc630d4f7fbe04a6f8e64b7874 2024-11-18T20:12:26,645 DEBUG [StoreCloser-TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/C/9f393cdd80ce414790fc9040d5f8fbcc to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/C/9f393cdd80ce414790fc9040d5f8fbcc 2024-11-18T20:12:26,646 DEBUG [StoreCloser-TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/C/74f1d8e339d342a9ade550c080960874 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/C/74f1d8e339d342a9ade550c080960874 2024-11-18T20:12:26,647 DEBUG [StoreCloser-TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/C/6ee05491a9b94a5c86dee6e8fd3fa393 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/C/6ee05491a9b94a5c86dee6e8fd3fa393 2024-11-18T20:12:26,648 DEBUG [StoreCloser-TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/C/427bbd64ca6947c9b52afa0d85685f9a to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/C/427bbd64ca6947c9b52afa0d85685f9a 2024-11-18T20:12:26,649 DEBUG [StoreCloser-TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/C/7c807d6029364cdf8693b06bc4cf0604 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/C/7c807d6029364cdf8693b06bc4cf0604 2024-11-18T20:12:26,650 DEBUG [StoreCloser-TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/C/afe362d8843742188e5c63ff3c9ce7d8 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/C/afe362d8843742188e5c63ff3c9ce7d8 2024-11-18T20:12:26,651 DEBUG [StoreCloser-TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/C/bd5835e1e6ac4244b2e746fac8937694 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/C/bd5835e1e6ac4244b2e746fac8937694 2024-11-18T20:12:26,652 DEBUG [StoreCloser-TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/C/bc865bcfd2744007984ced955360e6f3 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/C/bc865bcfd2744007984ced955360e6f3 2024-11-18T20:12:26,653 DEBUG [StoreCloser-TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/C/71f170f01f364dac9c12b546f53e20fa to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/C/71f170f01f364dac9c12b546f53e20fa 2024-11-18T20:12:26,655 DEBUG [StoreCloser-TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/C/ff309597fdc6400bb092706f52627c2e to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/C/ff309597fdc6400bb092706f52627c2e 2024-11-18T20:12:26,656 DEBUG [StoreCloser-TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/C/591d46c5d1434763a6c7c077da1dce7a to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/C/591d46c5d1434763a6c7c077da1dce7a 2024-11-18T20:12:26,658 DEBUG [StoreCloser-TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/C/56d32afce2e246c6baac00f288e84104 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/C/56d32afce2e246c6baac00f288e84104 2024-11-18T20:12:26,659 DEBUG [StoreCloser-TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/C/d4e8b147be684d3099d64db920d4e498 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/C/d4e8b147be684d3099d64db920d4e498 2024-11-18T20:12:26,660 DEBUG [StoreCloser-TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/C/0fcb7ac7831f4238a2d626df07f5cf08 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/C/0fcb7ac7831f4238a2d626df07f5cf08 2024-11-18T20:12:26,661 DEBUG [StoreCloser-TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/C/f472bbab37374b50b653dca80bbc51ac to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/C/f472bbab37374b50b653dca80bbc51ac 2024-11-18T20:12:26,662 DEBUG [StoreCloser-TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/C/be76e94061504f36b664767f51e08c86 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/C/be76e94061504f36b664767f51e08c86 2024-11-18T20:12:26,663 DEBUG [StoreCloser-TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/C/1a4e8da4738b4b9bb9fc00e6ff0346f4 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/C/1a4e8da4738b4b9bb9fc00e6ff0346f4 2024-11-18T20:12:26,665 DEBUG [StoreCloser-TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/C/30143802614f4f2d943e803bb5ada559 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/C/30143802614f4f2d943e803bb5ada559 2024-11-18T20:12:26,666 DEBUG [StoreCloser-TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/C/d318e0d36ef04bbfa81b4a9cec0bed57 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/C/d318e0d36ef04bbfa81b4a9cec0bed57 2024-11-18T20:12:26,667 DEBUG [StoreCloser-TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/C/4d511892eb394f2e87f47ae4755c8405 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/C/4d511892eb394f2e87f47ae4755c8405 2024-11-18T20:12:26,668 DEBUG [StoreCloser-TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/C/0bf801180970467380d807e7f4462a63 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/C/0bf801180970467380d807e7f4462a63 2024-11-18T20:12:26,669 DEBUG [StoreCloser-TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/C/ae65c7a9fa38497ca2777d173a4a27a4 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/C/ae65c7a9fa38497ca2777d173a4a27a4 2024-11-18T20:12:26,671 DEBUG [StoreCloser-TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/C/234cc462460544368a9fdfd2b1c17cfc to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/C/234cc462460544368a9fdfd2b1c17cfc 2024-11-18T20:12:26,672 DEBUG [StoreCloser-TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/C/79a7f500da114675bd904acea264b4fe to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/C/79a7f500da114675bd904acea264b4fe 2024-11-18T20:12:26,673 DEBUG [StoreCloser-TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/C/14c7cc34aa9f4feb9d610a1b8c652b4c to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/C/14c7cc34aa9f4feb9d610a1b8c652b4c 2024-11-18T20:12:26,675 DEBUG [StoreCloser-TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/C/6a4ad8aeebf541e3a78afbb395b5e0e3 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/C/6a4ad8aeebf541e3a78afbb395b5e0e3 2024-11-18T20:12:26,676 DEBUG [StoreCloser-TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/C/f6203fe303154cf8954019222620e9fd to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/C/f6203fe303154cf8954019222620e9fd 2024-11-18T20:12:26,682 DEBUG [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/recovered.edits/543.seqid, newMaxSeqId=543, maxSeqId=1 2024-11-18T20:12:26,683 INFO [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73. 2024-11-18T20:12:26,683 DEBUG [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1676): Region close journal for 326884d1d40acf488e8bd50aea838c73: Waiting for close lock at 1731960745230Running coprocessor pre-close hooks at 1731960745230Disabling compacts and flushes for region at 1731960745230Disabling writes for close at 1731960745231 (+1 ms)Obtaining lock to block concurrent updates at 1731960745231Preparing flush snapshotting stores in 326884d1d40acf488e8bd50aea838c73 at 1731960745231Finished memstore snapshotting TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=90720, getOffHeapSize=0, getCellsCount=750 at 1731960745231Flushing stores of TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73. at 1731960745232 (+1 ms)Flushing 326884d1d40acf488e8bd50aea838c73/A: creating writer at 1731960745233 (+1 ms)Flushing 326884d1d40acf488e8bd50aea838c73/A: appending metadata at 1731960745238 (+5 ms)Flushing 326884d1d40acf488e8bd50aea838c73/A: closing flushed file at 1731960745238Flushing 326884d1d40acf488e8bd50aea838c73/B: creating writer at 1731960745653 (+415 ms)Flushing 326884d1d40acf488e8bd50aea838c73/B: appending metadata at 1731960745656 (+3 ms)Flushing 326884d1d40acf488e8bd50aea838c73/B: closing flushed file at 1731960745656Flushing 326884d1d40acf488e8bd50aea838c73/C: creating writer at 1731960746066 (+410 ms)Flushing 326884d1d40acf488e8bd50aea838c73/C: appending metadata at 1731960746069 (+3 ms)Flushing 326884d1d40acf488e8bd50aea838c73/C: closing flushed file at 1731960746069Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@53695284: reopening flushed file at 1731960746480 (+411 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@ad30286: reopening flushed file at 1731960746486 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3acd3074: reopening flushed file at 1731960746494 (+8 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for 326884d1d40acf488e8bd50aea838c73 in 1270ms, sequenceid=540, compaction requested=true at 1731960746501 (+7 ms)Writing region close event to WAL at 1731960746678 (+177 ms)Running coprocessor post-close hooks at 1731960746683 (+5 ms)Closed at 1731960746683 2024-11-18T20:12:26,685 INFO [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] handler.UnassignRegionHandler(157): Closed 326884d1d40acf488e8bd50aea838c73 2024-11-18T20:12:26,686 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=326884d1d40acf488e8bd50aea838c73, regionState=CLOSED 2024-11-18T20:12:26,687 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=30, ppid=29, state=RUNNABLE, hasLock=false; CloseRegionProcedure 326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 because future has completed 2024-11-18T20:12:26,690 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=30, resume processing ppid=29 2024-11-18T20:12:26,690 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=30, ppid=29, state=SUCCESS, hasLock=false; CloseRegionProcedure 326884d1d40acf488e8bd50aea838c73, server=c0a89b2656d4,35963,1731960718223 in 1.6170 sec 2024-11-18T20:12:26,692 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=29, resume processing ppid=28 2024-11-18T20:12:26,692 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=29, ppid=28, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=326884d1d40acf488e8bd50aea838c73, UNASSIGN in 1.6250 sec 2024-11-18T20:12:26,695 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=28, resume processing ppid=27 2024-11-18T20:12:26,695 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=28, ppid=27, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.6330 sec 2024-11-18T20:12:26,697 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731960746696"}]},"ts":"1731960746696"} 2024-11-18T20:12:26,698 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-18T20:12:26,698 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-18T20:12:26,699 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=27, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.6480 sec 2024-11-18T20:12:27,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-18T20:12:27,213 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-18T20:12:27,217 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41277 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-18T20:12:27,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41277 {}] procedure2.ProcedureExecutor(1139): Stored pid=31, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-18T20:12:27,221 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=31, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-18T20:12:27,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-11-18T20:12:27,223 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=31, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-18T20:12:27,229 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73 2024-11-18T20:12:27,233 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/A, FileablePath, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/B, FileablePath, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/C, FileablePath, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/recovered.edits] 2024-11-18T20:12:27,237 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/A/2a1ada93f2ac49588ed65010244db0ba to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/A/2a1ada93f2ac49588ed65010244db0ba 2024-11-18T20:12:27,239 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/A/39c0fd20d09247e18aa3a0854dca1d0b to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/A/39c0fd20d09247e18aa3a0854dca1d0b 2024-11-18T20:12:27,241 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/A/3ab08836ff0f438fa80f0ddba9d6341f to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/A/3ab08836ff0f438fa80f0ddba9d6341f 2024-11-18T20:12:27,244 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/B/69c9420987944b9a807fde6a13b0bb76 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/B/69c9420987944b9a807fde6a13b0bb76 2024-11-18T20:12:27,245 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/B/ca27638f2c4345208df0f1e36e41490a to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/B/ca27638f2c4345208df0f1e36e41490a 2024-11-18T20:12:27,246 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/B/e8bb5f4af1ff4732914f997cb80879ad to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/B/e8bb5f4af1ff4732914f997cb80879ad 2024-11-18T20:12:27,249 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/C/0948f0f9726a45819bfcecaec4921cbd to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/C/0948f0f9726a45819bfcecaec4921cbd 2024-11-18T20:12:27,250 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/C/df42d6bcce78435ebdc53fa5de962093 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/C/df42d6bcce78435ebdc53fa5de962093 2024-11-18T20:12:27,251 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/C/e7cb4284c79040b9baadc4ae6233ceed to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/C/e7cb4284c79040b9baadc4ae6233ceed 2024-11-18T20:12:27,254 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/recovered.edits/543.seqid to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73/recovered.edits/543.seqid 2024-11-18T20:12:27,255 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/326884d1d40acf488e8bd50aea838c73 2024-11-18T20:12:27,255 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-18T20:12:27,262 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=31, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-18T20:12:27,266 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35963 {}] util.ReflectedFunctionCache(97): Populated cache for org.apache.hadoop.hbase.filter.KeyOnlyFilter in 0ms 2024-11-18T20:12:27,272 WARN [PEWorker-5 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-18T20:12:27,301 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-18T20:12:27,303 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=31, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-18T20:12:27,303 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-18T20:12:27,304 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731960747303"}]},"ts":"9223372036854775807"} 2024-11-18T20:12:27,308 INFO [PEWorker-5 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-18T20:12:27,308 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 326884d1d40acf488e8bd50aea838c73, NAME => 'TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73.', STARTKEY => '', ENDKEY => ''}] 2024-11-18T20:12:27,308 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-18T20:12:27,309 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731960747308"}]},"ts":"9223372036854775807"} 2024-11-18T20:12:27,312 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-18T20:12:27,313 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(133): Finished pid=31, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-18T20:12:27,314 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=31, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 96 msec 2024-11-18T20:12:27,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-11-18T20:12:27,334 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-18T20:12:27,334 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-18T20:12:27,345 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testMixedAtomicity Thread=238 (was 210) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-12 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_246716058_22 at /127.0.0.1:36794 [Waiting for operation #234] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-14 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1376622929_22 at /127.0.0.1:41312 [Waiting for operation #273] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_OPEN_REGION-regionserver/c0a89b2656d4:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-11 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-5 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-13 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-16 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-9 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS:0;c0a89b2656d4:35963-shortCompactions-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.PriorityBlockingQueue.take(PriorityBlockingQueue.java:535) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-15 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-10 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1376622929_22 at /127.0.0.1:36798 [Waiting for operation #235] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-8 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=455 (was 441) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=398 (was 312) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=3406 (was 3789) 2024-11-18T20:12:27,354 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testMobMixedAtomicity Thread=238, OpenFileDescriptor=455, MaxFileDescriptor=1048576, SystemLoadAverage=398, ProcessCount=11, AvailableMemoryMB=3402 2024-11-18T20:12:27,355 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41277 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-18T20:12:27,355 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41277 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-18T20:12:27,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41277 {}] procedure2.ProcedureExecutor(1139): Stored pid=32, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-18T20:12:27,358 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=32, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-18T20:12:27,358 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:27,358 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41277 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 32 2024-11-18T20:12:27,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-11-18T20:12:27,359 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=32, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-18T20:12:27,365 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073741956_1132 (size=959) 2024-11-18T20:12:27,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-11-18T20:12:27,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-11-18T20:12:27,768 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 9d5f6ceb1ccdf5b797c44f37788d88e8, NAME => 'TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123 2024-11-18T20:12:27,778 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073741957_1133 (size=53) 2024-11-18T20:12:27,922 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-18T20:12:27,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-11-18T20:12:28,180 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T20:12:28,180 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 9d5f6ceb1ccdf5b797c44f37788d88e8, disabling compactions & flushes 2024-11-18T20:12:28,180 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8. 2024-11-18T20:12:28,180 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8. 2024-11-18T20:12:28,180 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8. after waiting 0 ms 2024-11-18T20:12:28,180 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8. 2024-11-18T20:12:28,180 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8. 2024-11-18T20:12:28,180 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 9d5f6ceb1ccdf5b797c44f37788d88e8: Waiting for close lock at 1731960748180Disabling compacts and flushes for region at 1731960748180Disabling writes for close at 1731960748180Writing region close event to WAL at 1731960748180Closed at 1731960748180 2024-11-18T20:12:28,183 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=32, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-18T20:12:28,184 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731960748183"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731960748183"}]},"ts":"1731960748183"} 2024-11-18T20:12:28,187 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-18T20:12:28,188 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=32, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-18T20:12:28,188 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731960748188"}]},"ts":"1731960748188"} 2024-11-18T20:12:28,190 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-18T20:12:28,190 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=9d5f6ceb1ccdf5b797c44f37788d88e8, ASSIGN}] 2024-11-18T20:12:28,191 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=9d5f6ceb1ccdf5b797c44f37788d88e8, ASSIGN 2024-11-18T20:12:28,192 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=9d5f6ceb1ccdf5b797c44f37788d88e8, ASSIGN; state=OFFLINE, location=c0a89b2656d4,35963,1731960718223; forceNewPlan=false, retain=false 2024-11-18T20:12:28,343 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=33 updating hbase:meta row=9d5f6ceb1ccdf5b797c44f37788d88e8, regionState=OPENING, regionLocation=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:28,348 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=9d5f6ceb1ccdf5b797c44f37788d88e8, ASSIGN because future has completed 2024-11-18T20:12:28,349 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=34, ppid=33, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223}] 2024-11-18T20:12:28,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-11-18T20:12:28,512 INFO [RS_OPEN_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8. 2024-11-18T20:12:28,513 DEBUG [RS_OPEN_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(7752): Opening region: {ENCODED => 9d5f6ceb1ccdf5b797c44f37788d88e8, NAME => 'TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8.', STARTKEY => '', ENDKEY => ''} 2024-11-18T20:12:28,514 DEBUG [RS_OPEN_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 9d5f6ceb1ccdf5b797c44f37788d88e8 2024-11-18T20:12:28,514 DEBUG [RS_OPEN_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T20:12:28,514 DEBUG [RS_OPEN_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(7794): checking encryption for 9d5f6ceb1ccdf5b797c44f37788d88e8 2024-11-18T20:12:28,515 DEBUG [RS_OPEN_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(7797): checking classloading for 9d5f6ceb1ccdf5b797c44f37788d88e8 2024-11-18T20:12:28,517 INFO [StoreOpener-9d5f6ceb1ccdf5b797c44f37788d88e8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 9d5f6ceb1ccdf5b797c44f37788d88e8 2024-11-18T20:12:28,520 INFO [StoreOpener-9d5f6ceb1ccdf5b797c44f37788d88e8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9d5f6ceb1ccdf5b797c44f37788d88e8 columnFamilyName A 2024-11-18T20:12:28,520 DEBUG [StoreOpener-9d5f6ceb1ccdf5b797c44f37788d88e8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:28,521 INFO [StoreOpener-9d5f6ceb1ccdf5b797c44f37788d88e8-1 {}] regionserver.HStore(327): Store=9d5f6ceb1ccdf5b797c44f37788d88e8/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T20:12:28,521 INFO [StoreOpener-9d5f6ceb1ccdf5b797c44f37788d88e8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 9d5f6ceb1ccdf5b797c44f37788d88e8 2024-11-18T20:12:28,523 INFO [StoreOpener-9d5f6ceb1ccdf5b797c44f37788d88e8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9d5f6ceb1ccdf5b797c44f37788d88e8 columnFamilyName B 2024-11-18T20:12:28,523 DEBUG [StoreOpener-9d5f6ceb1ccdf5b797c44f37788d88e8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:28,524 INFO [StoreOpener-9d5f6ceb1ccdf5b797c44f37788d88e8-1 {}] regionserver.HStore(327): Store=9d5f6ceb1ccdf5b797c44f37788d88e8/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T20:12:28,524 INFO [StoreOpener-9d5f6ceb1ccdf5b797c44f37788d88e8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 9d5f6ceb1ccdf5b797c44f37788d88e8 2024-11-18T20:12:28,526 INFO [StoreOpener-9d5f6ceb1ccdf5b797c44f37788d88e8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9d5f6ceb1ccdf5b797c44f37788d88e8 columnFamilyName C 2024-11-18T20:12:28,526 DEBUG [StoreOpener-9d5f6ceb1ccdf5b797c44f37788d88e8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:28,526 INFO [StoreOpener-9d5f6ceb1ccdf5b797c44f37788d88e8-1 {}] regionserver.HStore(327): Store=9d5f6ceb1ccdf5b797c44f37788d88e8/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T20:12:28,527 DEBUG [RS_OPEN_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1038): replaying wal for 9d5f6ceb1ccdf5b797c44f37788d88e8 2024-11-18T20:12:28,528 DEBUG [RS_OPEN_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8 2024-11-18T20:12:28,528 DEBUG [RS_OPEN_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8 2024-11-18T20:12:28,529 DEBUG [RS_OPEN_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1048): stopping wal replay for 9d5f6ceb1ccdf5b797c44f37788d88e8 2024-11-18T20:12:28,529 DEBUG [RS_OPEN_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1060): Cleaning up temporary data for 9d5f6ceb1ccdf5b797c44f37788d88e8 2024-11-18T20:12:28,530 DEBUG [RS_OPEN_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-18T20:12:28,531 DEBUG [RS_OPEN_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1093): writing seq id for 9d5f6ceb1ccdf5b797c44f37788d88e8 2024-11-18T20:12:28,533 DEBUG [RS_OPEN_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-18T20:12:28,534 INFO [RS_OPEN_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1114): Opened 9d5f6ceb1ccdf5b797c44f37788d88e8; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64476443, jitterRate=-0.03922612965106964}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-18T20:12:28,534 DEBUG [RS_OPEN_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 9d5f6ceb1ccdf5b797c44f37788d88e8 2024-11-18T20:12:28,535 DEBUG [RS_OPEN_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1006): Region open journal for 9d5f6ceb1ccdf5b797c44f37788d88e8: Running coprocessor pre-open hook at 1731960748515Writing region info on filesystem at 1731960748515Initializing all the Stores at 1731960748517 (+2 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731960748517Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731960748517Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731960748517Cleaning up temporary data from old regions at 1731960748529 (+12 ms)Running coprocessor post-open hooks at 1731960748534 (+5 ms)Region opened successfully at 1731960748535 (+1 ms) 2024-11-18T20:12:28,536 INFO [RS_OPEN_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., pid=34, masterSystemTime=1731960748504 2024-11-18T20:12:28,538 DEBUG [RS_OPEN_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8. 2024-11-18T20:12:28,538 INFO [RS_OPEN_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8. 2024-11-18T20:12:28,538 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=33 updating hbase:meta row=9d5f6ceb1ccdf5b797c44f37788d88e8, regionState=OPEN, openSeqNum=2, regionLocation=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:28,540 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=34, ppid=33, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 because future has completed 2024-11-18T20:12:28,544 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=34, resume processing ppid=33 2024-11-18T20:12:28,544 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=34, ppid=33, state=SUCCESS, hasLock=false; OpenRegionProcedure 9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 in 193 msec 2024-11-18T20:12:28,546 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=33, resume processing ppid=32 2024-11-18T20:12:28,546 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=33, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=9d5f6ceb1ccdf5b797c44f37788d88e8, ASSIGN in 354 msec 2024-11-18T20:12:28,547 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=32, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-18T20:12:28,547 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731960748547"}]},"ts":"1731960748547"} 2024-11-18T20:12:28,549 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-18T20:12:28,550 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=32, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-18T20:12:28,552 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=32, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1950 sec 2024-11-18T20:12:29,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-11-18T20:12:29,504 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-18T20:12:29,506 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@49091041, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:12:29,506 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c0a89b2656d4,41277,-1 for getting cluster id 2024-11-18T20:12:29,506 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T20:12:29,508 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ac99df6b-7791-4693-a81a-6a5a39504490' 2024-11-18T20:12:29,508 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T20:12:29,508 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ac99df6b-7791-4693-a81a-6a5a39504490" 2024-11-18T20:12:29,509 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@11432ffc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:12:29,509 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c0a89b2656d4,41277,-1] 2024-11-18T20:12:29,509 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T20:12:29,510 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:12:29,512 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44558, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T20:12:29,513 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@746efc98, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:12:29,513 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T20:12:29,515 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c0a89b2656d4,35963,1731960718223, seqNum=-1] 2024-11-18T20:12:29,516 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T20:12:29,517 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47140, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T20:12:29,519 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-18T20:12:29,520 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncConnectionImpl(321): The fetched master address is c0a89b2656d4,41277,1731960717534 2024-11-18T20:12:29,520 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@7f6fc162 2024-11-18T20:12:29,520 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-18T20:12:29,521 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44570, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-18T20:12:29,526 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41277 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-18T20:12:29,526 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41277 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-18T20:12:29,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41277 {}] procedure2.ProcedureExecutor(1139): Stored pid=35, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-18T20:12:29,545 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073741958_1134 (size=995) 2024-11-18T20:12:29,947 DEBUG [PEWorker-5 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.959 2024-11-18T20:12:29,947 INFO [PEWorker-5 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.995 2024-11-18T20:12:29,950 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=36, ppid=35, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-18T20:12:29,957 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=37, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=9d5f6ceb1ccdf5b797c44f37788d88e8, REOPEN/MOVE}] 2024-11-18T20:12:29,958 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=37, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=9d5f6ceb1ccdf5b797c44f37788d88e8, REOPEN/MOVE 2024-11-18T20:12:29,958 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=37 updating hbase:meta row=9d5f6ceb1ccdf5b797c44f37788d88e8, regionState=CLOSING, regionLocation=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:29,960 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=37, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=9d5f6ceb1ccdf5b797c44f37788d88e8, REOPEN/MOVE because future has completed 2024-11-18T20:12:29,961 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-18T20:12:29,961 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=38, ppid=37, state=RUNNABLE, hasLock=false; CloseRegionProcedure 9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223}] 2024-11-18T20:12:30,114 INFO [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] handler.UnassignRegionHandler(122): Close 9d5f6ceb1ccdf5b797c44f37788d88e8 2024-11-18T20:12:30,114 DEBUG [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-18T20:12:30,114 DEBUG [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1722): Closing 9d5f6ceb1ccdf5b797c44f37788d88e8, disabling compactions & flushes 2024-11-18T20:12:30,114 INFO [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8. 2024-11-18T20:12:30,114 DEBUG [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8. 2024-11-18T20:12:30,114 DEBUG [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8. after waiting 0 ms 2024-11-18T20:12:30,114 DEBUG [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8. 2024-11-18T20:12:30,118 DEBUG [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-18T20:12:30,118 INFO [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8. 2024-11-18T20:12:30,118 DEBUG [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1676): Region close journal for 9d5f6ceb1ccdf5b797c44f37788d88e8: Waiting for close lock at 1731960750114Running coprocessor pre-close hooks at 1731960750114Disabling compacts and flushes for region at 1731960750114Disabling writes for close at 1731960750114Writing region close event to WAL at 1731960750114Running coprocessor post-close hooks at 1731960750118 (+4 ms)Closed at 1731960750118 2024-11-18T20:12:30,119 WARN [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegionServer(3299): Not adding moved region record: 9d5f6ceb1ccdf5b797c44f37788d88e8 to self. 2024-11-18T20:12:30,121 INFO [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] handler.UnassignRegionHandler(157): Closed 9d5f6ceb1ccdf5b797c44f37788d88e8 2024-11-18T20:12:30,121 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=37 updating hbase:meta row=9d5f6ceb1ccdf5b797c44f37788d88e8, regionState=CLOSED 2024-11-18T20:12:30,123 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=38, ppid=37, state=RUNNABLE, hasLock=false; CloseRegionProcedure 9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 because future has completed 2024-11-18T20:12:30,125 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=38, resume processing ppid=37 2024-11-18T20:12:30,125 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=38, ppid=37, state=SUCCESS, hasLock=false; CloseRegionProcedure 9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 in 163 msec 2024-11-18T20:12:30,126 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=37, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=9d5f6ceb1ccdf5b797c44f37788d88e8, REOPEN/MOVE; state=CLOSED, location=c0a89b2656d4,35963,1731960718223; forceNewPlan=false, retain=true 2024-11-18T20:12:30,276 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=37 updating hbase:meta row=9d5f6ceb1ccdf5b797c44f37788d88e8, regionState=OPENING, regionLocation=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:30,278 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=37, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=9d5f6ceb1ccdf5b797c44f37788d88e8, REOPEN/MOVE because future has completed 2024-11-18T20:12:30,278 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=39, ppid=37, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223}] 2024-11-18T20:12:30,435 INFO [RS_OPEN_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8. 2024-11-18T20:12:30,435 DEBUG [RS_OPEN_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(7752): Opening region: {ENCODED => 9d5f6ceb1ccdf5b797c44f37788d88e8, NAME => 'TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8.', STARTKEY => '', ENDKEY => ''} 2024-11-18T20:12:30,435 DEBUG [RS_OPEN_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 9d5f6ceb1ccdf5b797c44f37788d88e8 2024-11-18T20:12:30,435 DEBUG [RS_OPEN_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T20:12:30,436 DEBUG [RS_OPEN_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(7794): checking encryption for 9d5f6ceb1ccdf5b797c44f37788d88e8 2024-11-18T20:12:30,436 DEBUG [RS_OPEN_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(7797): checking classloading for 9d5f6ceb1ccdf5b797c44f37788d88e8 2024-11-18T20:12:30,442 INFO [StoreOpener-9d5f6ceb1ccdf5b797c44f37788d88e8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 9d5f6ceb1ccdf5b797c44f37788d88e8 2024-11-18T20:12:30,450 INFO [StoreOpener-9d5f6ceb1ccdf5b797c44f37788d88e8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9d5f6ceb1ccdf5b797c44f37788d88e8 columnFamilyName A 2024-11-18T20:12:30,453 DEBUG [StoreOpener-9d5f6ceb1ccdf5b797c44f37788d88e8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:30,453 INFO [StoreOpener-9d5f6ceb1ccdf5b797c44f37788d88e8-1 {}] regionserver.HStore(327): Store=9d5f6ceb1ccdf5b797c44f37788d88e8/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T20:12:30,454 INFO [StoreOpener-9d5f6ceb1ccdf5b797c44f37788d88e8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 9d5f6ceb1ccdf5b797c44f37788d88e8 2024-11-18T20:12:30,455 INFO [StoreOpener-9d5f6ceb1ccdf5b797c44f37788d88e8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9d5f6ceb1ccdf5b797c44f37788d88e8 columnFamilyName B 2024-11-18T20:12:30,455 DEBUG [StoreOpener-9d5f6ceb1ccdf5b797c44f37788d88e8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:30,455 INFO [StoreOpener-9d5f6ceb1ccdf5b797c44f37788d88e8-1 {}] regionserver.HStore(327): Store=9d5f6ceb1ccdf5b797c44f37788d88e8/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T20:12:30,456 INFO [StoreOpener-9d5f6ceb1ccdf5b797c44f37788d88e8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 9d5f6ceb1ccdf5b797c44f37788d88e8 2024-11-18T20:12:30,456 INFO [StoreOpener-9d5f6ceb1ccdf5b797c44f37788d88e8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9d5f6ceb1ccdf5b797c44f37788d88e8 columnFamilyName C 2024-11-18T20:12:30,456 DEBUG [StoreOpener-9d5f6ceb1ccdf5b797c44f37788d88e8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:30,457 INFO [StoreOpener-9d5f6ceb1ccdf5b797c44f37788d88e8-1 {}] regionserver.HStore(327): Store=9d5f6ceb1ccdf5b797c44f37788d88e8/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T20:12:30,457 DEBUG [RS_OPEN_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1038): replaying wal for 9d5f6ceb1ccdf5b797c44f37788d88e8 2024-11-18T20:12:30,457 DEBUG [RS_OPEN_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8 2024-11-18T20:12:30,458 DEBUG [RS_OPEN_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8 2024-11-18T20:12:30,459 DEBUG [RS_OPEN_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1048): stopping wal replay for 9d5f6ceb1ccdf5b797c44f37788d88e8 2024-11-18T20:12:30,459 DEBUG [RS_OPEN_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1060): Cleaning up temporary data for 9d5f6ceb1ccdf5b797c44f37788d88e8 2024-11-18T20:12:30,460 DEBUG [RS_OPEN_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-18T20:12:30,461 DEBUG [RS_OPEN_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1093): writing seq id for 9d5f6ceb1ccdf5b797c44f37788d88e8 2024-11-18T20:12:30,462 INFO [RS_OPEN_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1114): Opened 9d5f6ceb1ccdf5b797c44f37788d88e8; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72280749, jitterRate=0.07706709206104279}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-18T20:12:30,462 DEBUG [RS_OPEN_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 9d5f6ceb1ccdf5b797c44f37788d88e8 2024-11-18T20:12:30,463 DEBUG [RS_OPEN_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1006): Region open journal for 9d5f6ceb1ccdf5b797c44f37788d88e8: Running coprocessor pre-open hook at 1731960750436Writing region info on filesystem at 1731960750436Initializing all the Stores at 1731960750437 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731960750437Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731960750438 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731960750438Cleaning up temporary data from old regions at 1731960750459 (+21 ms)Running coprocessor post-open hooks at 1731960750462 (+3 ms)Region opened successfully at 1731960750463 (+1 ms) 2024-11-18T20:12:30,463 INFO [RS_OPEN_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., pid=39, masterSystemTime=1731960750430 2024-11-18T20:12:30,465 DEBUG [RS_OPEN_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8. 2024-11-18T20:12:30,465 INFO [RS_OPEN_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8. 2024-11-18T20:12:30,466 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=37 updating hbase:meta row=9d5f6ceb1ccdf5b797c44f37788d88e8, regionState=OPEN, openSeqNum=5, regionLocation=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:30,467 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=39, ppid=37, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 because future has completed 2024-11-18T20:12:30,470 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=39, resume processing ppid=37 2024-11-18T20:12:30,470 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=39, ppid=37, state=SUCCESS, hasLock=false; OpenRegionProcedure 9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 in 190 msec 2024-11-18T20:12:30,472 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=37, resume processing ppid=36 2024-11-18T20:12:30,472 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=37, ppid=36, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=9d5f6ceb1ccdf5b797c44f37788d88e8, REOPEN/MOVE in 513 msec 2024-11-18T20:12:30,476 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=36, resume processing ppid=35 2024-11-18T20:12:30,476 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=36, ppid=35, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 523 msec 2024-11-18T20:12:30,479 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=35, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 948 msec 2024-11-18T20:12:30,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=35 2024-11-18T20:12:30,481 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-18T20:12:30,482 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@cce2794, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:12:30,482 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c0a89b2656d4,41277,-1 for getting cluster id 2024-11-18T20:12:30,482 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T20:12:30,483 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ac99df6b-7791-4693-a81a-6a5a39504490' 2024-11-18T20:12:30,483 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T20:12:30,483 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ac99df6b-7791-4693-a81a-6a5a39504490" 2024-11-18T20:12:30,483 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@182d538d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:12:30,484 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c0a89b2656d4,41277,-1] 2024-11-18T20:12:30,484 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T20:12:30,484 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:12:30,485 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44598, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T20:12:30,485 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@799faa56, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:12:30,486 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2f683619, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:12:30,486 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c0a89b2656d4,41277,-1 for getting cluster id 2024-11-18T20:12:30,486 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T20:12:30,487 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ac99df6b-7791-4693-a81a-6a5a39504490' 2024-11-18T20:12:30,487 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T20:12:30,487 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ac99df6b-7791-4693-a81a-6a5a39504490" 2024-11-18T20:12:30,488 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@f3fb0ca, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:12:30,488 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c0a89b2656d4,41277,-1] 2024-11-18T20:12:30,488 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T20:12:30,488 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:12:30,489 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44614, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T20:12:30,489 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7c90cf87, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:12:30,490 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@dd47c94, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:12:30,490 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c0a89b2656d4,41277,-1 for getting cluster id 2024-11-18T20:12:30,491 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T20:12:30,491 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ac99df6b-7791-4693-a81a-6a5a39504490' 2024-11-18T20:12:30,491 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T20:12:30,491 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ac99df6b-7791-4693-a81a-6a5a39504490" 2024-11-18T20:12:30,492 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@159910ad, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:12:30,492 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c0a89b2656d4,41277,-1] 2024-11-18T20:12:30,492 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T20:12:30,492 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:12:30,493 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44642, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T20:12:30,493 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5844c102, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:12:30,494 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@51d91e8d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:12:30,494 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c0a89b2656d4,41277,-1 for getting cluster id 2024-11-18T20:12:30,494 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T20:12:30,495 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ac99df6b-7791-4693-a81a-6a5a39504490' 2024-11-18T20:12:30,495 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T20:12:30,495 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ac99df6b-7791-4693-a81a-6a5a39504490" 2024-11-18T20:12:30,495 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6017a0a3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:12:30,495 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c0a89b2656d4,41277,-1] 2024-11-18T20:12:30,495 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T20:12:30,496 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:12:30,496 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44646, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T20:12:30,497 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3a0ea88e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:12:30,498 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@85a5f7a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:12:30,498 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c0a89b2656d4,41277,-1 for getting cluster id 2024-11-18T20:12:30,498 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T20:12:30,499 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ac99df6b-7791-4693-a81a-6a5a39504490' 2024-11-18T20:12:30,499 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T20:12:30,499 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ac99df6b-7791-4693-a81a-6a5a39504490" 2024-11-18T20:12:30,499 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5cba15f5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:12:30,499 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c0a89b2656d4,41277,-1] 2024-11-18T20:12:30,499 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T20:12:30,499 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:12:30,500 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44664, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T20:12:30,501 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@a7980bc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:12:30,502 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7c77b203, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:12:30,502 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c0a89b2656d4,41277,-1 for getting cluster id 2024-11-18T20:12:30,502 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T20:12:30,503 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ac99df6b-7791-4693-a81a-6a5a39504490' 2024-11-18T20:12:30,503 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T20:12:30,503 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ac99df6b-7791-4693-a81a-6a5a39504490" 2024-11-18T20:12:30,503 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@11b482e2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:12:30,503 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c0a89b2656d4,41277,-1] 2024-11-18T20:12:30,504 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T20:12:30,504 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:12:30,504 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44672, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T20:12:30,505 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@74e5c62, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:12:30,506 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6ab1fb6e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:12:30,506 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c0a89b2656d4,41277,-1 for getting cluster id 2024-11-18T20:12:30,506 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T20:12:30,506 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ac99df6b-7791-4693-a81a-6a5a39504490' 2024-11-18T20:12:30,507 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T20:12:30,507 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ac99df6b-7791-4693-a81a-6a5a39504490" 2024-11-18T20:12:30,507 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@539855c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:12:30,507 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c0a89b2656d4,41277,-1] 2024-11-18T20:12:30,507 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T20:12:30,507 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:12:30,508 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44690, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T20:12:30,508 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@825702d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:12:30,509 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6dd18334, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:12:30,509 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c0a89b2656d4,41277,-1 for getting cluster id 2024-11-18T20:12:30,509 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T20:12:30,510 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ac99df6b-7791-4693-a81a-6a5a39504490' 2024-11-18T20:12:30,510 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T20:12:30,510 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ac99df6b-7791-4693-a81a-6a5a39504490" 2024-11-18T20:12:30,510 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@64c5d935, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:12:30,510 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c0a89b2656d4,41277,-1] 2024-11-18T20:12:30,510 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T20:12:30,510 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:12:30,511 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44702, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T20:12:30,512 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@20e82b1b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:12:30,512 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@52c57d01, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:12:30,512 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c0a89b2656d4,41277,-1 for getting cluster id 2024-11-18T20:12:30,513 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T20:12:30,513 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ac99df6b-7791-4693-a81a-6a5a39504490' 2024-11-18T20:12:30,513 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T20:12:30,514 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ac99df6b-7791-4693-a81a-6a5a39504490" 2024-11-18T20:12:30,514 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7d1d7a02, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:12:30,514 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c0a89b2656d4,41277,-1] 2024-11-18T20:12:30,514 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T20:12:30,514 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:12:30,515 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44726, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T20:12:30,516 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@364eb416, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:12:30,516 DEBUG [Thread-680 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T20:12:30,517 DEBUG [Thread-678 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T20:12:30,517 DEBUG [Thread-682 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T20:12:30,517 DEBUG [Thread-671 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T20:12:30,517 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c0a89b2656d4,35963,1731960718223, seqNum=-1] 2024-11-18T20:12:30,517 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-18T20:12:30,517 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c0a89b2656d4,35963,1731960718223, seqNum=-1] 2024-11-18T20:12:30,517 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c0a89b2656d4,35963,1731960718223, seqNum=-1] 2024-11-18T20:12:30,517 DEBUG [Thread-667 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T20:12:30,518 DEBUG [Thread-684 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T20:12:30,519 DEBUG [Thread-675 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T20:12:30,518 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c0a89b2656d4,35963,1731960718223, seqNum=-1] 2024-11-18T20:12:30,518 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T20:12:30,519 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c0a89b2656d4,35963,1731960718223, seqNum=-1] 2024-11-18T20:12:30,519 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T20:12:30,519 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T20:12:30,518 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T20:12:30,518 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T20:12:30,520 DEBUG [Thread-669 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T20:12:30,520 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47146, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T20:12:30,520 DEBUG [Thread-673 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T20:12:30,521 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47178, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T20:12:30,521 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c0a89b2656d4,35963,1731960718223, seqNum=-1] 2024-11-18T20:12:30,521 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47158, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T20:12:30,521 DEBUG [Thread-669 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c0a89b2656d4,35963,1731960718223, seqNum=-1] 2024-11-18T20:12:30,521 DEBUG [Thread-673 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c0a89b2656d4,35963,1731960718223, seqNum=-1] 2024-11-18T20:12:30,521 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c0a89b2656d4,35963,1731960718223, seqNum=-1] 2024-11-18T20:12:30,521 DEBUG [Thread-669 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T20:12:30,521 DEBUG [Thread-673 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T20:12:30,521 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T20:12:30,522 DEBUG [Thread-684 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T20:12:30,524 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47166, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T20:12:30,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] procedure2.ProcedureExecutor(1139): Stored pid=40, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees 2024-11-18T20:12:30,524 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5] 2024-11-18T20:12:30,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-18T20:12:30,526 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47182, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T20:12:30,527 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=40, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-18T20:12:30,527 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47206, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T20:12:30,527 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47208, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T20:12:30,527 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5] 2024-11-18T20:12:30,529 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5] 2024-11-18T20:12:30,529 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47184, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T20:12:30,529 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5] 2024-11-18T20:12:30,530 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47220, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T20:12:30,530 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5] 2024-11-18T20:12:30,530 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=40, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-18T20:12:30,530 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=41, ppid=40, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-18T20:12:30,531 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5] 2024-11-18T20:12:30,533 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5] 2024-11-18T20:12:30,535 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5] 2024-11-18T20:12:30,535 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5] 2024-11-18T20:12:30,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(8855): Flush requested on 9d5f6ceb1ccdf5b797c44f37788d88e8 2024-11-18T20:12:30,545 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-18T20:12:30,546 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9d5f6ceb1ccdf5b797c44f37788d88e8 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-18T20:12:30,606 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411189de5b8dd2869434f91d4a501b1541fbe_9d5f6ceb1ccdf5b797c44f37788d88e8 is 50, key is test_row_0/A:col10/1731960750537/Put/seqid=0 2024-11-18T20:12:30,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-18T20:12:30,633 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:30,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47184 deadline: 1731960810615, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:30,634 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:30,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47220 deadline: 1731960810620, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:30,634 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:30,634 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:30,635 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:12:30,635 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:30,635 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:30,635 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:12:30,635 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:30,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47206 deadline: 1731960810622, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:30,635 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:30,635 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:30,635 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:30,635 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:12:30,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47158 deadline: 1731960810617, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:30,636 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:30,636 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:30,636 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:12:30,642 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073741959_1135 (size=14594) 2024-11-18T20:12:30,643 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:30,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47182 deadline: 1731960810624, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:30,644 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:30,644 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:30,644 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:12:30,683 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35963 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-18T20:12:30,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8. 2024-11-18T20:12:30,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8. as already flushing 2024-11-18T20:12:30,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8. 2024-11-18T20:12:30,684 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 9d5f6ceb1ccdf5b797c44f37788d88e8, NAME => 'TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:12:30,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 9d5f6ceb1ccdf5b797c44f37788d88e8, NAME => 'TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:12:30,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9d5f6ceb1ccdf5b797c44f37788d88e8, NAME => 'TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9d5f6ceb1ccdf5b797c44f37788d88e8, NAME => 'TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:12:30,751 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:30,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47220 deadline: 1731960810743, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:30,752 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:30,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47184 deadline: 1731960810743, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:30,752 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:30,752 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:30,752 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:12:30,752 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:30,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47158 deadline: 1731960810744, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:30,752 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:30,753 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:30,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47206 deadline: 1731960810744, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:30,753 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:30,753 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:12:30,753 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:30,753 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:30,753 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:12:30,754 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:30,754 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:30,754 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:12:30,765 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:30,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47182 deadline: 1731960810754, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:30,766 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:30,766 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:30,766 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:12:30,840 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35963 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-18T20:12:30,840 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8. 2024-11-18T20:12:30,840 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8. as already flushing 2024-11-18T20:12:30,840 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8. 2024-11-18T20:12:30,841 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 9d5f6ceb1ccdf5b797c44f37788d88e8, NAME => 'TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:12:30,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 9d5f6ceb1ccdf5b797c44f37788d88e8, NAME => 'TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:12:30,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-18T20:12:30,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9d5f6ceb1ccdf5b797c44f37788d88e8, NAME => 'TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9d5f6ceb1ccdf5b797c44f37788d88e8, NAME => 'TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:12:30,967 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:30,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47220 deadline: 1731960810963, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:30,968 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:30,968 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:30,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47206 deadline: 1731960810964, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:30,968 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:30,968 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:12:30,968 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:30,968 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:30,968 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:30,968 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:12:30,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47158 deadline: 1731960810964, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:30,969 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:30,969 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:30,969 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:30,969 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:12:30,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47184 deadline: 1731960810964, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:30,969 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:30,969 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:30,969 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:12:30,976 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:30,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47182 deadline: 1731960810973, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:30,978 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:30,978 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:30,978 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:12:31,002 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35963 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-18T20:12:31,002 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8. 2024-11-18T20:12:31,002 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8. as already flushing 2024-11-18T20:12:31,002 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8. 2024-11-18T20:12:31,002 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 9d5f6ceb1ccdf5b797c44f37788d88e8, NAME => 'TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:12:31,002 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 9d5f6ceb1ccdf5b797c44f37788d88e8, NAME => 'TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:12:31,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9d5f6ceb1ccdf5b797c44f37788d88e8, NAME => 'TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9d5f6ceb1ccdf5b797c44f37788d88e8, NAME => 'TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:12:31,044 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:31,050 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411189de5b8dd2869434f91d4a501b1541fbe_9d5f6ceb1ccdf5b797c44f37788d88e8 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411189de5b8dd2869434f91d4a501b1541fbe_9d5f6ceb1ccdf5b797c44f37788d88e8 2024-11-18T20:12:31,051 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/.tmp/A/c0f2285265cb49309b85d2fa7aec3ad1, store: [table=TestAcidGuarantees family=A region=9d5f6ceb1ccdf5b797c44f37788d88e8] 2024-11-18T20:12:31,061 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/.tmp/A/c0f2285265cb49309b85d2fa7aec3ad1 is 175, key is test_row_0/A:col10/1731960750537/Put/seqid=0 2024-11-18T20:12:31,080 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073741960_1136 (size=39549) 2024-11-18T20:12:31,083 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=16, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/.tmp/A/c0f2285265cb49309b85d2fa7aec3ad1 2024-11-18T20:12:31,133 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/.tmp/B/f1fbc10d3bee4ddc91795b98e07ec8a9 is 50, key is test_row_0/B:col10/1731960750537/Put/seqid=0 2024-11-18T20:12:31,142 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073741961_1137 (size=12001) 2024-11-18T20:12:31,143 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/.tmp/B/f1fbc10d3bee4ddc91795b98e07ec8a9 2024-11-18T20:12:31,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-18T20:12:31,157 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35963 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-18T20:12:31,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8. 2024-11-18T20:12:31,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8. as already flushing 2024-11-18T20:12:31,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8. 2024-11-18T20:12:31,157 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 9d5f6ceb1ccdf5b797c44f37788d88e8, NAME => 'TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:12:31,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 9d5f6ceb1ccdf5b797c44f37788d88e8, NAME => 'TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:12:31,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9d5f6ceb1ccdf5b797c44f37788d88e8, NAME => 'TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9d5f6ceb1ccdf5b797c44f37788d88e8, NAME => 'TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:12:31,203 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/.tmp/C/a237f2ac7f5b43be98189738aaad1223 is 50, key is test_row_0/C:col10/1731960750537/Put/seqid=0 2024-11-18T20:12:31,251 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073741962_1138 (size=12001) 2024-11-18T20:12:31,252 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/.tmp/C/a237f2ac7f5b43be98189738aaad1223 2024-11-18T20:12:31,264 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/.tmp/A/c0f2285265cb49309b85d2fa7aec3ad1 as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/A/c0f2285265cb49309b85d2fa7aec3ad1 2024-11-18T20:12:31,275 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/A/c0f2285265cb49309b85d2fa7aec3ad1, entries=200, sequenceid=16, filesize=38.6 K 2024-11-18T20:12:31,275 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:31,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47184 deadline: 1731960811273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:31,276 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:31,276 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:31,276 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:31,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47206 deadline: 1731960811273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:31,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47158 deadline: 1731960811273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:31,276 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:31,276 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:12:31,276 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:31,276 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:31,276 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:12:31,277 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:31,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47220 deadline: 1731960811275, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:31,278 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:31,278 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:31,278 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:31,278 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:31,278 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:12:31,278 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:12:31,285 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/.tmp/B/f1fbc10d3bee4ddc91795b98e07ec8a9 as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/B/f1fbc10d3bee4ddc91795b98e07ec8a9 2024-11-18T20:12:31,295 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/B/f1fbc10d3bee4ddc91795b98e07ec8a9, entries=150, sequenceid=16, filesize=11.7 K 2024-11-18T20:12:31,297 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/.tmp/C/a237f2ac7f5b43be98189738aaad1223 as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/C/a237f2ac7f5b43be98189738aaad1223 2024-11-18T20:12:31,303 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:31,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47182 deadline: 1731960811284, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:31,304 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:31,304 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:31,304 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:12:31,311 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/C/a237f2ac7f5b43be98189738aaad1223, entries=150, sequenceid=16, filesize=11.7 K 2024-11-18T20:12:31,311 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35963 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-18T20:12:31,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8. 2024-11-18T20:12:31,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8. as already flushing 2024-11-18T20:12:31,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8. 2024-11-18T20:12:31,312 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 9d5f6ceb1ccdf5b797c44f37788d88e8, NAME => 'TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:12:31,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 9d5f6ceb1ccdf5b797c44f37788d88e8, NAME => 'TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:12:31,312 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 9d5f6ceb1ccdf5b797c44f37788d88e8 in 767ms, sequenceid=16, compaction requested=false 2024-11-18T20:12:31,313 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9d5f6ceb1ccdf5b797c44f37788d88e8: 2024-11-18T20:12:31,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9d5f6ceb1ccdf5b797c44f37788d88e8, NAME => 'TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9d5f6ceb1ccdf5b797c44f37788d88e8, NAME => 'TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:12:31,465 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35963 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-18T20:12:31,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8. 2024-11-18T20:12:31,465 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2902): Flushing 9d5f6ceb1ccdf5b797c44f37788d88e8 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-18T20:12:31,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118f7752a19b8d14e7badba705d75c56465_9d5f6ceb1ccdf5b797c44f37788d88e8 is 50, key is test_row_0/A:col10/1731960750607/Put/seqid=0 2024-11-18T20:12:31,512 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073741963_1139 (size=12154) 2024-11-18T20:12:31,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:31,520 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118f7752a19b8d14e7badba705d75c56465_9d5f6ceb1ccdf5b797c44f37788d88e8 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118f7752a19b8d14e7badba705d75c56465_9d5f6ceb1ccdf5b797c44f37788d88e8 2024-11-18T20:12:31,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/.tmp/A/c9f67eefaab1486680e5a996b6d8817b, store: [table=TestAcidGuarantees family=A region=9d5f6ceb1ccdf5b797c44f37788d88e8] 2024-11-18T20:12:31,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/.tmp/A/c9f67eefaab1486680e5a996b6d8817b is 175, key is test_row_0/A:col10/1731960750607/Put/seqid=0 2024-11-18T20:12:31,570 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073741964_1140 (size=30955) 2024-11-18T20:12:31,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-18T20:12:31,787 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8. as already flushing 2024-11-18T20:12:31,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(8855): Flush requested on 9d5f6ceb1ccdf5b797c44f37788d88e8 2024-11-18T20:12:31,806 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:31,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47206 deadline: 1731960811802, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:31,807 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:31,807 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:31,807 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:12:31,808 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:31,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47220 deadline: 1731960811804, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:31,808 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:31,808 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:31,808 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:31,808 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:12:31,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47158 deadline: 1731960811806, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:31,809 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:31,809 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:31,809 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:12:31,809 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:31,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47184 deadline: 1731960811807, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:31,810 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:31,810 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:31,810 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:12:31,820 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:31,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47182 deadline: 1731960811814, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:31,821 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:31,821 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:31,821 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:12:31,918 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:31,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47206 deadline: 1731960811913, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:31,919 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:31,919 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:31,919 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:31,919 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:12:31,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47220 deadline: 1731960811913, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:31,919 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:31,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47184 deadline: 1731960811914, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:31,920 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:31,920 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:31,920 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:31,920 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:12:31,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47158 deadline: 1731960811914, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:31,920 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:31,920 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:31,920 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:12:31,920 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:31,920 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:31,920 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:12:31,973 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=40, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/.tmp/A/c9f67eefaab1486680e5a996b6d8817b 2024-11-18T20:12:31,986 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/.tmp/B/e8a7cc11d67b421f933954581e2cc452 is 50, key is test_row_0/B:col10/1731960750607/Put/seqid=0 2024-11-18T20:12:31,994 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073741965_1141 (size=12001) 2024-11-18T20:12:31,995 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=40 (bloomFilter=true), to=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/.tmp/B/e8a7cc11d67b421f933954581e2cc452 2024-11-18T20:12:32,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/.tmp/C/e4072932f97e4bc9aa4ea0dab2795a1d is 50, key is test_row_0/C:col10/1731960750607/Put/seqid=0 2024-11-18T20:12:32,035 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073741966_1142 (size=12001) 2024-11-18T20:12:32,035 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=40 (bloomFilter=true), to=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/.tmp/C/e4072932f97e4bc9aa4ea0dab2795a1d 2024-11-18T20:12:32,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/.tmp/A/c9f67eefaab1486680e5a996b6d8817b as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/A/c9f67eefaab1486680e5a996b6d8817b 2024-11-18T20:12:32,055 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/A/c9f67eefaab1486680e5a996b6d8817b, entries=150, sequenceid=40, filesize=30.2 K 2024-11-18T20:12:32,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/.tmp/B/e8a7cc11d67b421f933954581e2cc452 as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/B/e8a7cc11d67b421f933954581e2cc452 2024-11-18T20:12:32,065 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/B/e8a7cc11d67b421f933954581e2cc452, entries=150, sequenceid=40, filesize=11.7 K 2024-11-18T20:12:32,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/.tmp/C/e4072932f97e4bc9aa4ea0dab2795a1d as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/C/e4072932f97e4bc9aa4ea0dab2795a1d 2024-11-18T20:12:32,084 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/C/e4072932f97e4bc9aa4ea0dab2795a1d, entries=150, sequenceid=40, filesize=11.7 K 2024-11-18T20:12:32,085 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for 9d5f6ceb1ccdf5b797c44f37788d88e8 in 620ms, sequenceid=40, compaction requested=false 2024-11-18T20:12:32,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2603): Flush status journal for 9d5f6ceb1ccdf5b797c44f37788d88e8: 2024-11-18T20:12:32,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8. 2024-11-18T20:12:32,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=41 2024-11-18T20:12:32,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.HMaster(4169): Remote procedure done, pid=41 2024-11-18T20:12:32,089 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=41, resume processing ppid=40 2024-11-18T20:12:32,089 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=41, ppid=40, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5570 sec 2024-11-18T20:12:32,094 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=40, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees in 1.5750 sec 2024-11-18T20:12:32,135 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-18T20:12:32,135 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9d5f6ceb1ccdf5b797c44f37788d88e8 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-18T20:12:32,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(8855): Flush requested on 9d5f6ceb1ccdf5b797c44f37788d88e8 2024-11-18T20:12:32,150 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118b359682701214fa79b9c8fb5d4626a8e_9d5f6ceb1ccdf5b797c44f37788d88e8 is 50, key is test_row_0/A:col10/1731960751787/Put/seqid=0 2024-11-18T20:12:32,173 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:32,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47184 deadline: 1731960812167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:32,174 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:32,174 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:32,174 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:32,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47220 deadline: 1731960812169, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:32,174 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:12:32,174 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:32,174 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:32,174 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:32,174 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:12:32,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47158 deadline: 1731960812170, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:32,175 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:32,175 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:32,175 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:12:32,177 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073741967_1143 (size=12154) 2024-11-18T20:12:32,177 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:32,179 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:32,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47206 deadline: 1731960812174, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:32,180 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:32,180 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:32,180 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:12:32,184 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118b359682701214fa79b9c8fb5d4626a8e_9d5f6ceb1ccdf5b797c44f37788d88e8 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118b359682701214fa79b9c8fb5d4626a8e_9d5f6ceb1ccdf5b797c44f37788d88e8 2024-11-18T20:12:32,189 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/.tmp/A/2a02f4728fc24b718d041e65260cc24c, store: [table=TestAcidGuarantees family=A region=9d5f6ceb1ccdf5b797c44f37788d88e8] 2024-11-18T20:12:32,190 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/.tmp/A/2a02f4728fc24b718d041e65260cc24c is 175, key is test_row_0/A:col10/1731960751787/Put/seqid=0 2024-11-18T20:12:32,199 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073741968_1144 (size=30955) 2024-11-18T20:12:32,201 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=54, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/.tmp/A/2a02f4728fc24b718d041e65260cc24c 2024-11-18T20:12:32,216 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/.tmp/B/347b868f5fa84793b0ef2f608bfe6462 is 50, key is test_row_0/B:col10/1731960751787/Put/seqid=0 2024-11-18T20:12:32,233 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073741969_1145 (size=12001) 2024-11-18T20:12:32,235 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=54 (bloomFilter=true), to=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/.tmp/B/347b868f5fa84793b0ef2f608bfe6462 2024-11-18T20:12:32,249 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/.tmp/C/5ec448785187415e8bbaa060c1064b04 is 50, key is test_row_0/C:col10/1731960751787/Put/seqid=0 2024-11-18T20:12:32,260 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-18T20:12:32,278 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073741970_1146 (size=12001) 2024-11-18T20:12:32,288 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:32,289 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:32,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47220 deadline: 1731960812283, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:32,289 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:32,289 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:32,289 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:12:32,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47184 deadline: 1731960812283, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:32,290 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:32,290 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:32,290 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:32,290 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:12:32,290 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:32,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47158 deadline: 1731960812284, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:32,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47206 deadline: 1731960812284, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:32,290 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:32,290 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:32,290 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:12:32,290 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:32,290 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:32,291 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:12:32,498 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:32,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47220 deadline: 1731960812493, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:32,498 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:32,498 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:32,499 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:32,499 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:12:32,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47184 deadline: 1731960812494, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:32,499 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:32,499 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:32,499 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:32,499 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:12:32,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47158 deadline: 1731960812494, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:32,499 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:32,500 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:32,499 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:32,500 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:12:32,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47206 deadline: 1731960812494, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:32,500 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:32,500 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:32,500 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:12:32,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-18T20:12:32,674 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-18T20:12:32,675 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-18T20:12:32,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] procedure2.ProcedureExecutor(1139): Stored pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees 2024-11-18T20:12:32,677 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-18T20:12:32,678 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-18T20:12:32,678 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=43, ppid=42, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-18T20:12:32,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-18T20:12:32,682 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=54 (bloomFilter=true), to=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/.tmp/C/5ec448785187415e8bbaa060c1064b04 2024-11-18T20:12:32,690 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/.tmp/A/2a02f4728fc24b718d041e65260cc24c as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/A/2a02f4728fc24b718d041e65260cc24c 2024-11-18T20:12:32,697 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/A/2a02f4728fc24b718d041e65260cc24c, entries=150, sequenceid=54, filesize=30.2 K 2024-11-18T20:12:32,698 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/.tmp/B/347b868f5fa84793b0ef2f608bfe6462 as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/B/347b868f5fa84793b0ef2f608bfe6462 2024-11-18T20:12:32,704 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/B/347b868f5fa84793b0ef2f608bfe6462, entries=150, sequenceid=54, filesize=11.7 K 2024-11-18T20:12:32,705 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/.tmp/C/5ec448785187415e8bbaa060c1064b04 as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/C/5ec448785187415e8bbaa060c1064b04 2024-11-18T20:12:32,710 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/C/5ec448785187415e8bbaa060c1064b04, entries=150, sequenceid=54, filesize=11.7 K 2024-11-18T20:12:32,712 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for 9d5f6ceb1ccdf5b797c44f37788d88e8 in 577ms, sequenceid=54, compaction requested=true 2024-11-18T20:12:32,712 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9d5f6ceb1ccdf5b797c44f37788d88e8: 2024-11-18T20:12:32,712 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9d5f6ceb1ccdf5b797c44f37788d88e8:A, priority=-2147483648, current under compaction store size is 1 2024-11-18T20:12:32,712 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:12:32,712 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T20:12:32,712 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9d5f6ceb1ccdf5b797c44f37788d88e8:B, priority=-2147483648, current under compaction store size is 2 2024-11-18T20:12:32,713 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:12:32,713 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T20:12:32,713 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9d5f6ceb1ccdf5b797c44f37788d88e8:C, priority=-2147483648, current under compaction store size is 3 2024-11-18T20:12:32,713 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T20:12:32,715 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T20:12:32,715 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101459 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T20:12:32,715 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HStore(1541): 9d5f6ceb1ccdf5b797c44f37788d88e8/B is initiating minor compaction (all files) 2024-11-18T20:12:32,715 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HStore(1541): 9d5f6ceb1ccdf5b797c44f37788d88e8/A is initiating minor compaction (all files) 2024-11-18T20:12:32,715 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9d5f6ceb1ccdf5b797c44f37788d88e8/B in TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8. 2024-11-18T20:12:32,715 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/B/f1fbc10d3bee4ddc91795b98e07ec8a9, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/B/e8a7cc11d67b421f933954581e2cc452, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/B/347b868f5fa84793b0ef2f608bfe6462] into tmpdir=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/.tmp, totalSize=35.2 K 2024-11-18T20:12:32,717 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9d5f6ceb1ccdf5b797c44f37788d88e8/A in TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8. 2024-11-18T20:12:32,717 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/A/c0f2285265cb49309b85d2fa7aec3ad1, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/A/c9f67eefaab1486680e5a996b6d8817b, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/A/2a02f4728fc24b718d041e65260cc24c] into tmpdir=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/.tmp, totalSize=99.1 K 2024-11-18T20:12:32,717 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8. 2024-11-18T20:12:32,717 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8. files: [hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/A/c0f2285265cb49309b85d2fa7aec3ad1, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/A/c9f67eefaab1486680e5a996b6d8817b, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/A/2a02f4728fc24b718d041e65260cc24c] 2024-11-18T20:12:32,719 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.Compactor(225): Compacting f1fbc10d3bee4ddc91795b98e07ec8a9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1731960750537 2024-11-18T20:12:32,720 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.Compactor(225): Compacting e8a7cc11d67b421f933954581e2cc452, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1731960750607 2024-11-18T20:12:32,720 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.Compactor(225): Compacting 347b868f5fa84793b0ef2f608bfe6462, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1731960751787 2024-11-18T20:12:32,720 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] compactions.Compactor(225): Compacting c0f2285265cb49309b85d2fa7aec3ad1, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1731960750537 2024-11-18T20:12:32,721 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] compactions.Compactor(225): Compacting c9f67eefaab1486680e5a996b6d8817b, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1731960750607 2024-11-18T20:12:32,721 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2a02f4728fc24b718d041e65260cc24c, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1731960751787 2024-11-18T20:12:32,740 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9d5f6ceb1ccdf5b797c44f37788d88e8#B#compaction#128 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:12:32,740 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/.tmp/B/dd5c5ee28e48453bb106b3662079efef is 50, key is test_row_0/B:col10/1731960751787/Put/seqid=0 2024-11-18T20:12:32,746 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=9d5f6ceb1ccdf5b797c44f37788d88e8] 2024-11-18T20:12:32,770 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241118429bc41636464722b76f240624b1b68b_9d5f6ceb1ccdf5b797c44f37788d88e8 store=[table=TestAcidGuarantees family=A region=9d5f6ceb1ccdf5b797c44f37788d88e8] 2024-11-18T20:12:32,775 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241118429bc41636464722b76f240624b1b68b_9d5f6ceb1ccdf5b797c44f37788d88e8, store=[table=TestAcidGuarantees family=A region=9d5f6ceb1ccdf5b797c44f37788d88e8] 2024-11-18T20:12:32,775 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118429bc41636464722b76f240624b1b68b_9d5f6ceb1ccdf5b797c44f37788d88e8 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=9d5f6ceb1ccdf5b797c44f37788d88e8] 2024-11-18T20:12:32,779 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073741971_1147 (size=12104) 2024-11-18T20:12:32,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-18T20:12:32,790 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/.tmp/B/dd5c5ee28e48453bb106b3662079efef as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/B/dd5c5ee28e48453bb106b3662079efef 2024-11-18T20:12:32,800 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9d5f6ceb1ccdf5b797c44f37788d88e8/B of 9d5f6ceb1ccdf5b797c44f37788d88e8 into dd5c5ee28e48453bb106b3662079efef(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:12:32,800 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9d5f6ceb1ccdf5b797c44f37788d88e8: 2024-11-18T20:12:32,800 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., storeName=9d5f6ceb1ccdf5b797c44f37788d88e8/B, priority=13, startTime=1731960752712; duration=0sec 2024-11-18T20:12:32,800 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T20:12:32,801 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9d5f6ceb1ccdf5b797c44f37788d88e8:B 2024-11-18T20:12:32,801 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T20:12:32,803 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T20:12:32,803 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HStore(1541): 9d5f6ceb1ccdf5b797c44f37788d88e8/C is initiating minor compaction (all files) 2024-11-18T20:12:32,803 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9d5f6ceb1ccdf5b797c44f37788d88e8/C in TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8. 2024-11-18T20:12:32,803 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/C/a237f2ac7f5b43be98189738aaad1223, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/C/e4072932f97e4bc9aa4ea0dab2795a1d, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/C/5ec448785187415e8bbaa060c1064b04] into tmpdir=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/.tmp, totalSize=35.2 K 2024-11-18T20:12:32,804 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.Compactor(225): Compacting a237f2ac7f5b43be98189738aaad1223, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1731960750537 2024-11-18T20:12:32,805 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.Compactor(225): Compacting e4072932f97e4bc9aa4ea0dab2795a1d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1731960750607 2024-11-18T20:12:32,805 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.Compactor(225): Compacting 5ec448785187415e8bbaa060c1064b04, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1731960751787 2024-11-18T20:12:32,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(8855): Flush requested on 9d5f6ceb1ccdf5b797c44f37788d88e8 2024-11-18T20:12:32,810 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-18T20:12:32,810 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9d5f6ceb1ccdf5b797c44f37788d88e8 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-18T20:12:32,818 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073741972_1148 (size=4469) 2024-11-18T20:12:32,820 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9d5f6ceb1ccdf5b797c44f37788d88e8#A#compaction#129 average throughput is 0.33 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:12:32,820 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/.tmp/A/156bcc007c634c6fa7315ad53c7623d7 is 175, key is test_row_0/A:col10/1731960751787/Put/seqid=0 2024-11-18T20:12:32,827 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9d5f6ceb1ccdf5b797c44f37788d88e8#C#compaction#130 average throughput is 1.31 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:12:32,827 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/.tmp/C/e1bedff458a04273817b88fcd6019fe7 is 50, key is test_row_0/C:col10/1731960751787/Put/seqid=0 2024-11-18T20:12:32,831 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35963 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-18T20:12:32,831 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8. 2024-11-18T20:12:32,831 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:32,831 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8. as already flushing 2024-11-18T20:12:32,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47220 deadline: 1731960812827, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:32,831 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8. 2024-11-18T20:12:32,831 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 9d5f6ceb1ccdf5b797c44f37788d88e8, NAME => 'TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:12:32,832 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 9d5f6ceb1ccdf5b797c44f37788d88e8, NAME => 'TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:12:32,832 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:32,832 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:32,832 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:32,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47184 deadline: 1731960812829, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:32,832 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:12:32,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9d5f6ceb1ccdf5b797c44f37788d88e8, NAME => 'TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9d5f6ceb1ccdf5b797c44f37788d88e8, NAME => 'TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:12:32,833 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:32,833 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:32,833 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:12:32,838 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:32,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47206 deadline: 1731960812832, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:32,838 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:32,838 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:32,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47158 deadline: 1731960812832, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:32,838 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:32,838 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:12:32,839 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:32,839 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:32,839 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:32,839 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:12:32,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47182 deadline: 1731960812833, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:32,839 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to c0a89b2656d4,35963,1731960718223 for 'test_row_2' in 9d5f6ceb1ccdf5b797c44f37788d88e8 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2216 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-18T20:12:32,840 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:32,840 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:32,840 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:12:32,845 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118ce67afc2c80c423a91e2193366e4f2b9_9d5f6ceb1ccdf5b797c44f37788d88e8 is 50, key is test_row_0/A:col10/1731960752166/Put/seqid=0 2024-11-18T20:12:32,850 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073741973_1149 (size=31058) 2024-11-18T20:12:32,865 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073741974_1150 (size=12104) 2024-11-18T20:12:32,868 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/.tmp/A/156bcc007c634c6fa7315ad53c7623d7 as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/A/156bcc007c634c6fa7315ad53c7623d7 2024-11-18T20:12:32,878 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/.tmp/C/e1bedff458a04273817b88fcd6019fe7 as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/C/e1bedff458a04273817b88fcd6019fe7 2024-11-18T20:12:32,886 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9d5f6ceb1ccdf5b797c44f37788d88e8/A of 9d5f6ceb1ccdf5b797c44f37788d88e8 into 156bcc007c634c6fa7315ad53c7623d7(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:12:32,886 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9d5f6ceb1ccdf5b797c44f37788d88e8: 2024-11-18T20:12:32,886 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., storeName=9d5f6ceb1ccdf5b797c44f37788d88e8/A, priority=13, startTime=1731960752712; duration=0sec 2024-11-18T20:12:32,887 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:12:32,887 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9d5f6ceb1ccdf5b797c44f37788d88e8:A 2024-11-18T20:12:32,888 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9d5f6ceb1ccdf5b797c44f37788d88e8/C of 9d5f6ceb1ccdf5b797c44f37788d88e8 into e1bedff458a04273817b88fcd6019fe7(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:12:32,888 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9d5f6ceb1ccdf5b797c44f37788d88e8: 2024-11-18T20:12:32,888 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., storeName=9d5f6ceb1ccdf5b797c44f37788d88e8/C, priority=13, startTime=1731960752713; duration=0sec 2024-11-18T20:12:32,888 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:12:32,889 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9d5f6ceb1ccdf5b797c44f37788d88e8:C 2024-11-18T20:12:32,892 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073741975_1151 (size=14594) 2024-11-18T20:12:32,893 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:32,899 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118ce67afc2c80c423a91e2193366e4f2b9_9d5f6ceb1ccdf5b797c44f37788d88e8 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118ce67afc2c80c423a91e2193366e4f2b9_9d5f6ceb1ccdf5b797c44f37788d88e8 2024-11-18T20:12:32,901 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/.tmp/A/237de5ed6437485fb888eeb558983e15, store: [table=TestAcidGuarantees family=A region=9d5f6ceb1ccdf5b797c44f37788d88e8] 2024-11-18T20:12:32,902 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/.tmp/A/237de5ed6437485fb888eeb558983e15 is 175, key is test_row_0/A:col10/1731960752166/Put/seqid=0 2024-11-18T20:12:32,919 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073741976_1152 (size=39549) 2024-11-18T20:12:32,920 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=80, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/.tmp/A/237de5ed6437485fb888eeb558983e15 2024-11-18T20:12:32,937 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:32,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47220 deadline: 1731960812933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:32,937 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:32,937 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:32,938 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:12:32,943 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:32,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47184 deadline: 1731960812943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:32,944 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:32,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47206 deadline: 1731960812943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:32,943 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/.tmp/B/850fb40d28654286a3b8a66c3657b85d is 50, key is test_row_0/B:col10/1731960752166/Put/seqid=0 2024-11-18T20:12:32,944 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:32,944 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:32,944 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:32,944 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:32,944 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:32,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47158 deadline: 1731960812943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:32,944 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:12:32,944 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:12:32,945 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:32,945 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:32,945 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:12:32,959 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073741977_1153 (size=12001) 2024-11-18T20:12:32,984 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35963 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-18T20:12:32,989 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8. 2024-11-18T20:12:32,989 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8. as already flushing 2024-11-18T20:12:32,989 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8. 2024-11-18T20:12:32,989 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 9d5f6ceb1ccdf5b797c44f37788d88e8, NAME => 'TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:12:32,990 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 9d5f6ceb1ccdf5b797c44f37788d88e8, NAME => 'TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:12:32,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9d5f6ceb1ccdf5b797c44f37788d88e8, NAME => 'TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9d5f6ceb1ccdf5b797c44f37788d88e8, NAME => 'TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:12:32,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-18T20:12:33,143 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35963 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-18T20:12:33,143 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8. 2024-11-18T20:12:33,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8. as already flushing 2024-11-18T20:12:33,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8. 2024-11-18T20:12:33,144 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 9d5f6ceb1ccdf5b797c44f37788d88e8, NAME => 'TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:12:33,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 9d5f6ceb1ccdf5b797c44f37788d88e8, NAME => 'TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:12:33,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9d5f6ceb1ccdf5b797c44f37788d88e8, NAME => 'TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9d5f6ceb1ccdf5b797c44f37788d88e8, NAME => 'TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:12:33,147 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:33,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47220 deadline: 1731960813143, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:33,147 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:33,147 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:33,147 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:12:33,154 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:33,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47184 deadline: 1731960813153, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:33,155 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:33,155 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:33,155 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:33,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47206 deadline: 1731960813153, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:33,155 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:12:33,155 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:33,155 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:33,155 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:12:33,157 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:33,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47158 deadline: 1731960813153, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:33,158 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:33,158 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:33,158 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:12:33,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-18T20:12:33,310 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35963 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-18T20:12:33,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8. 2024-11-18T20:12:33,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8. as already flushing 2024-11-18T20:12:33,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8. 2024-11-18T20:12:33,311 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 9d5f6ceb1ccdf5b797c44f37788d88e8, NAME => 'TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:12:33,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 9d5f6ceb1ccdf5b797c44f37788d88e8, NAME => 'TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:12:33,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9d5f6ceb1ccdf5b797c44f37788d88e8, NAME => 'TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9d5f6ceb1ccdf5b797c44f37788d88e8, NAME => 'TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:12:33,360 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=80 (bloomFilter=true), to=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/.tmp/B/850fb40d28654286a3b8a66c3657b85d 2024-11-18T20:12:33,373 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/.tmp/C/d26895fa6d6040d1b7d3b944241ec251 is 50, key is test_row_0/C:col10/1731960752166/Put/seqid=0 2024-11-18T20:12:33,397 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073741978_1154 (size=12001) 2024-11-18T20:12:33,399 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=80 (bloomFilter=true), to=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/.tmp/C/d26895fa6d6040d1b7d3b944241ec251 2024-11-18T20:12:33,409 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/.tmp/A/237de5ed6437485fb888eeb558983e15 as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/A/237de5ed6437485fb888eeb558983e15 2024-11-18T20:12:33,422 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/A/237de5ed6437485fb888eeb558983e15, entries=200, sequenceid=80, filesize=38.6 K 2024-11-18T20:12:33,424 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/.tmp/B/850fb40d28654286a3b8a66c3657b85d as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/B/850fb40d28654286a3b8a66c3657b85d 2024-11-18T20:12:33,432 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/B/850fb40d28654286a3b8a66c3657b85d, entries=150, sequenceid=80, filesize=11.7 K 2024-11-18T20:12:33,435 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/.tmp/C/d26895fa6d6040d1b7d3b944241ec251 as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/C/d26895fa6d6040d1b7d3b944241ec251 2024-11-18T20:12:33,451 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/C/d26895fa6d6040d1b7d3b944241ec251, entries=150, sequenceid=80, filesize=11.7 K 2024-11-18T20:12:33,458 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 9d5f6ceb1ccdf5b797c44f37788d88e8 in 648ms, sequenceid=80, compaction requested=false 2024-11-18T20:12:33,459 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9d5f6ceb1ccdf5b797c44f37788d88e8: 2024-11-18T20:12:33,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(8855): Flush requested on 9d5f6ceb1ccdf5b797c44f37788d88e8 2024-11-18T20:12:33,461 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-18T20:12:33,462 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9d5f6ceb1ccdf5b797c44f37788d88e8 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-18T20:12:33,467 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35963 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-18T20:12:33,467 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8. 2024-11-18T20:12:33,467 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8. as already flushing 2024-11-18T20:12:33,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8. 2024-11-18T20:12:33,468 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 9d5f6ceb1ccdf5b797c44f37788d88e8, NAME => 'TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:12:33,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 9d5f6ceb1ccdf5b797c44f37788d88e8, NAME => 'TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:12:33,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9d5f6ceb1ccdf5b797c44f37788d88e8, NAME => 'TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9d5f6ceb1ccdf5b797c44f37788d88e8, NAME => 'TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:12:33,488 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118bb346766f8064b62b611b4fb3f31e19f_9d5f6ceb1ccdf5b797c44f37788d88e8 is 50, key is test_row_0/A:col10/1731960753459/Put/seqid=0 2024-11-18T20:12:33,535 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073741979_1155 (size=14594) 2024-11-18T20:12:33,536 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:33,546 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118bb346766f8064b62b611b4fb3f31e19f_9d5f6ceb1ccdf5b797c44f37788d88e8 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118bb346766f8064b62b611b4fb3f31e19f_9d5f6ceb1ccdf5b797c44f37788d88e8 2024-11-18T20:12:33,548 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/.tmp/A/ea6d38a7cd30486f98515c556d750e84, store: [table=TestAcidGuarantees family=A region=9d5f6ceb1ccdf5b797c44f37788d88e8] 2024-11-18T20:12:33,548 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/.tmp/A/ea6d38a7cd30486f98515c556d750e84 is 175, key is test_row_0/A:col10/1731960753459/Put/seqid=0 2024-11-18T20:12:33,562 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:33,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47184 deadline: 1731960813546, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:33,562 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:33,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47220 deadline: 1731960813546, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:33,563 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:33,563 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:33,563 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:12:33,564 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:33,564 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:33,564 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:12:33,566 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:33,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47206 deadline: 1731960813562, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:33,567 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:33,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47158 deadline: 1731960813562, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:33,567 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:33,567 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:33,567 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:33,567 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:33,567 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:12:33,567 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:12:33,587 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073741980_1156 (size=39549) 2024-11-18T20:12:33,589 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=94, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/.tmp/A/ea6d38a7cd30486f98515c556d750e84 2024-11-18T20:12:33,615 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/.tmp/B/591cbd36f0384cc4aba373b134f75759 is 50, key is test_row_0/B:col10/1731960753459/Put/seqid=0 2024-11-18T20:12:33,621 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35963 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-18T20:12:33,622 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8. 2024-11-18T20:12:33,622 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8. as already flushing 2024-11-18T20:12:33,622 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8. 2024-11-18T20:12:33,622 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 9d5f6ceb1ccdf5b797c44f37788d88e8, NAME => 'TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:12:33,622 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 9d5f6ceb1ccdf5b797c44f37788d88e8, NAME => 'TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:12:33,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9d5f6ceb1ccdf5b797c44f37788d88e8, NAME => 'TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9d5f6ceb1ccdf5b797c44f37788d88e8, NAME => 'TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:12:33,666 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073741981_1157 (size=12001) 2024-11-18T20:12:33,680 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:33,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47206 deadline: 1731960813673, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:33,682 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:33,682 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:33,682 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:33,682 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:12:33,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47220 deadline: 1731960813673, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:33,682 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:33,682 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:33,682 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:33,682 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:12:33,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47158 deadline: 1731960813673, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:33,683 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:33,683 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:33,683 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:33,683 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:12:33,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47184 deadline: 1731960813673, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:33,683 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:33,684 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:33,684 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:12:33,775 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35963 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-18T20:12:33,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8. 2024-11-18T20:12:33,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8. as already flushing 2024-11-18T20:12:33,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8. 2024-11-18T20:12:33,776 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 9d5f6ceb1ccdf5b797c44f37788d88e8, NAME => 'TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:12:33,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 9d5f6ceb1ccdf5b797c44f37788d88e8, NAME => 'TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:12:33,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9d5f6ceb1ccdf5b797c44f37788d88e8, NAME => 'TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9d5f6ceb1ccdf5b797c44f37788d88e8, NAME => 'TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:12:33,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-18T20:12:33,898 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:33,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47206 deadline: 1731960813894, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:33,899 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:33,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47220 deadline: 1731960813894, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:33,899 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:33,900 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:33,900 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:12:33,900 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:33,900 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:33,900 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:12:33,900 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:33,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47158 deadline: 1731960813894, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:33,900 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:33,900 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:33,900 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:33,900 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:12:33,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47184 deadline: 1731960813894, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:33,901 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:33,901 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:33,901 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:12:33,930 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35963 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-18T20:12:33,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8. 2024-11-18T20:12:33,931 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8. as already flushing 2024-11-18T20:12:33,931 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8. 2024-11-18T20:12:33,931 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 9d5f6ceb1ccdf5b797c44f37788d88e8, NAME => 'TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:12:33,931 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 9d5f6ceb1ccdf5b797c44f37788d88e8, NAME => 'TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:12:33,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9d5f6ceb1ccdf5b797c44f37788d88e8, NAME => 'TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9d5f6ceb1ccdf5b797c44f37788d88e8, NAME => 'TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:12:34,068 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=94 (bloomFilter=true), to=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/.tmp/B/591cbd36f0384cc4aba373b134f75759 2024-11-18T20:12:34,086 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35963 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-18T20:12:34,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8. 2024-11-18T20:12:34,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8. as already flushing 2024-11-18T20:12:34,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8. 2024-11-18T20:12:34,087 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 9d5f6ceb1ccdf5b797c44f37788d88e8, NAME => 'TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:12:34,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 9d5f6ceb1ccdf5b797c44f37788d88e8, NAME => 'TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:12:34,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9d5f6ceb1ccdf5b797c44f37788d88e8, NAME => 'TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9d5f6ceb1ccdf5b797c44f37788d88e8, NAME => 'TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:12:34,105 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/.tmp/C/c8b0082833a84d63aedd266b9c332cb7 is 50, key is test_row_0/C:col10/1731960753459/Put/seqid=0 2024-11-18T20:12:34,148 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073741982_1158 (size=12001) 2024-11-18T20:12:34,150 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=94 (bloomFilter=true), to=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/.tmp/C/c8b0082833a84d63aedd266b9c332cb7 2024-11-18T20:12:34,158 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/.tmp/A/ea6d38a7cd30486f98515c556d750e84 as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/A/ea6d38a7cd30486f98515c556d750e84 2024-11-18T20:12:34,165 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/A/ea6d38a7cd30486f98515c556d750e84, entries=200, sequenceid=94, filesize=38.6 K 2024-11-18T20:12:34,167 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/.tmp/B/591cbd36f0384cc4aba373b134f75759 as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/B/591cbd36f0384cc4aba373b134f75759 2024-11-18T20:12:34,177 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/B/591cbd36f0384cc4aba373b134f75759, entries=150, sequenceid=94, filesize=11.7 K 2024-11-18T20:12:34,179 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/.tmp/C/c8b0082833a84d63aedd266b9c332cb7 as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/C/c8b0082833a84d63aedd266b9c332cb7 2024-11-18T20:12:34,187 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/C/c8b0082833a84d63aedd266b9c332cb7, entries=150, sequenceid=94, filesize=11.7 K 2024-11-18T20:12:34,191 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=147.60 KB/151140 for 9d5f6ceb1ccdf5b797c44f37788d88e8 in 730ms, sequenceid=94, compaction requested=true 2024-11-18T20:12:34,191 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9d5f6ceb1ccdf5b797c44f37788d88e8: 2024-11-18T20:12:34,191 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9d5f6ceb1ccdf5b797c44f37788d88e8:A, priority=-2147483648, current under compaction store size is 1 2024-11-18T20:12:34,191 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:12:34,191 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T20:12:34,191 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9d5f6ceb1ccdf5b797c44f37788d88e8:B, priority=-2147483648, current under compaction store size is 2 2024-11-18T20:12:34,191 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:12:34,191 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T20:12:34,191 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9d5f6ceb1ccdf5b797c44f37788d88e8:C, priority=-2147483648, current under compaction store size is 3 2024-11-18T20:12:34,193 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T20:12:34,193 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 110156 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T20:12:34,193 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HStore(1541): 9d5f6ceb1ccdf5b797c44f37788d88e8/A is initiating minor compaction (all files) 2024-11-18T20:12:34,193 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HStore(1541): 9d5f6ceb1ccdf5b797c44f37788d88e8/B is initiating minor compaction (all files) 2024-11-18T20:12:34,193 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9d5f6ceb1ccdf5b797c44f37788d88e8/A in TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8. 2024-11-18T20:12:34,193 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9d5f6ceb1ccdf5b797c44f37788d88e8/B in TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8. 2024-11-18T20:12:34,193 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/B/dd5c5ee28e48453bb106b3662079efef, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/B/850fb40d28654286a3b8a66c3657b85d, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/B/591cbd36f0384cc4aba373b134f75759] into tmpdir=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/.tmp, totalSize=35.3 K 2024-11-18T20:12:34,194 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/A/156bcc007c634c6fa7315ad53c7623d7, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/A/237de5ed6437485fb888eeb558983e15, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/A/ea6d38a7cd30486f98515c556d750e84] into tmpdir=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/.tmp, totalSize=107.6 K 2024-11-18T20:12:34,194 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8. 2024-11-18T20:12:34,194 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8. files: [hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/A/156bcc007c634c6fa7315ad53c7623d7, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/A/237de5ed6437485fb888eeb558983e15, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/A/ea6d38a7cd30486f98515c556d750e84] 2024-11-18T20:12:34,194 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] compactions.Compactor(225): Compacting dd5c5ee28e48453bb106b3662079efef, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1731960751787 2024-11-18T20:12:34,194 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.Compactor(225): Compacting 156bcc007c634c6fa7315ad53c7623d7, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1731960751787 2024-11-18T20:12:34,195 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] compactions.Compactor(225): Compacting 850fb40d28654286a3b8a66c3657b85d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=80, earliestPutTs=1731960752166 2024-11-18T20:12:34,195 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.Compactor(225): Compacting 237de5ed6437485fb888eeb558983e15, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=80, earliestPutTs=1731960752166 2024-11-18T20:12:34,195 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T20:12:34,196 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] compactions.Compactor(225): Compacting 591cbd36f0384cc4aba373b134f75759, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1731960752824 2024-11-18T20:12:34,196 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.Compactor(225): Compacting ea6d38a7cd30486f98515c556d750e84, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1731960752824 2024-11-18T20:12:34,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(8855): Flush requested on 9d5f6ceb1ccdf5b797c44f37788d88e8 2024-11-18T20:12:34,206 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-18T20:12:34,206 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9d5f6ceb1ccdf5b797c44f37788d88e8 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-18T20:12:34,221 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=9d5f6ceb1ccdf5b797c44f37788d88e8] 2024-11-18T20:12:34,222 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:34,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47220 deadline: 1731960814215, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:34,223 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:34,223 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:34,223 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:12:34,224 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:34,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47158 deadline: 1731960814217, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:34,224 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:34,224 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:34,224 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:12:34,227 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:34,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47206 deadline: 1731960814221, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:34,228 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:34,228 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:34,228 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:34,228 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:12:34,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47184 deadline: 1731960814222, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:34,229 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:34,229 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:34,229 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:12:34,239 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9d5f6ceb1ccdf5b797c44f37788d88e8#B#compaction#138 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:12:34,240 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/.tmp/B/105a322f94b241759e166e95d61e1a9b is 50, key is test_row_0/B:col10/1731960753459/Put/seqid=0 2024-11-18T20:12:34,240 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35963 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-18T20:12:34,241 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8. 2024-11-18T20:12:34,241 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8. as already flushing 2024-11-18T20:12:34,241 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8. 2024-11-18T20:12:34,241 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 9d5f6ceb1ccdf5b797c44f37788d88e8, NAME => 'TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:12:34,241 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 9d5f6ceb1ccdf5b797c44f37788d88e8, NAME => 'TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:12:34,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9d5f6ceb1ccdf5b797c44f37788d88e8, NAME => 'TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9d5f6ceb1ccdf5b797c44f37788d88e8, NAME => 'TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:12:34,247 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241118174f559a91aa4f74a5a468bfc915dd16_9d5f6ceb1ccdf5b797c44f37788d88e8 store=[table=TestAcidGuarantees family=A region=9d5f6ceb1ccdf5b797c44f37788d88e8] 2024-11-18T20:12:34,251 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241118174f559a91aa4f74a5a468bfc915dd16_9d5f6ceb1ccdf5b797c44f37788d88e8, store=[table=TestAcidGuarantees family=A region=9d5f6ceb1ccdf5b797c44f37788d88e8] 2024-11-18T20:12:34,252 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118174f559a91aa4f74a5a468bfc915dd16_9d5f6ceb1ccdf5b797c44f37788d88e8 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=9d5f6ceb1ccdf5b797c44f37788d88e8] 2024-11-18T20:12:34,277 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118f80ff673c5344be2b5497315e3334ff6_9d5f6ceb1ccdf5b797c44f37788d88e8 is 50, key is test_row_0/A:col10/1731960753547/Put/seqid=0 2024-11-18T20:12:34,295 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073741983_1159 (size=12207) 2024-11-18T20:12:34,312 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/.tmp/B/105a322f94b241759e166e95d61e1a9b as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/B/105a322f94b241759e166e95d61e1a9b 2024-11-18T20:12:34,324 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9d5f6ceb1ccdf5b797c44f37788d88e8/B of 9d5f6ceb1ccdf5b797c44f37788d88e8 into 105a322f94b241759e166e95d61e1a9b(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:12:34,324 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9d5f6ceb1ccdf5b797c44f37788d88e8: 2024-11-18T20:12:34,324 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., storeName=9d5f6ceb1ccdf5b797c44f37788d88e8/B, priority=13, startTime=1731960754191; duration=0sec 2024-11-18T20:12:34,324 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T20:12:34,324 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9d5f6ceb1ccdf5b797c44f37788d88e8:B 2024-11-18T20:12:34,324 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T20:12:34,330 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T20:12:34,330 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HStore(1541): 9d5f6ceb1ccdf5b797c44f37788d88e8/C is initiating minor compaction (all files) 2024-11-18T20:12:34,330 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9d5f6ceb1ccdf5b797c44f37788d88e8/C in TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8. 2024-11-18T20:12:34,330 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/C/e1bedff458a04273817b88fcd6019fe7, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/C/d26895fa6d6040d1b7d3b944241ec251, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/C/c8b0082833a84d63aedd266b9c332cb7] into tmpdir=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/.tmp, totalSize=35.3 K 2024-11-18T20:12:34,332 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] compactions.Compactor(225): Compacting e1bedff458a04273817b88fcd6019fe7, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1731960751787 2024-11-18T20:12:34,332 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] compactions.Compactor(225): Compacting d26895fa6d6040d1b7d3b944241ec251, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=80, earliestPutTs=1731960752166 2024-11-18T20:12:34,333 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] compactions.Compactor(225): Compacting c8b0082833a84d63aedd266b9c332cb7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1731960752824 2024-11-18T20:12:34,335 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073741984_1160 (size=4469) 2024-11-18T20:12:34,340 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:34,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47220 deadline: 1731960814333, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:34,340 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:34,340 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:34,340 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:34,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47158 deadline: 1731960814334, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:34,340 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:12:34,341 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:34,341 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:34,341 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:34,341 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:12:34,343 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073741985_1161 (size=17034) 2024-11-18T20:12:34,345 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:34,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47206 deadline: 1731960814334, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:34,346 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:34,346 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:34,346 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:12:34,346 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:34,354 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118f80ff673c5344be2b5497315e3334ff6_9d5f6ceb1ccdf5b797c44f37788d88e8 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118f80ff673c5344be2b5497315e3334ff6_9d5f6ceb1ccdf5b797c44f37788d88e8 2024-11-18T20:12:34,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47184 deadline: 1731960814334, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:34,357 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:34,357 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:34,357 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:12:34,363 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/.tmp/A/82936db65f6b4993b25aa32eac0b21d7, store: [table=TestAcidGuarantees family=A region=9d5f6ceb1ccdf5b797c44f37788d88e8] 2024-11-18T20:12:34,363 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/.tmp/A/82936db65f6b4993b25aa32eac0b21d7 is 175, key is test_row_0/A:col10/1731960753547/Put/seqid=0 2024-11-18T20:12:34,374 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9d5f6ceb1ccdf5b797c44f37788d88e8#C#compaction#140 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:12:34,376 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/.tmp/C/e8c4015a7e704d1eb3191eef0cd574a9 is 50, key is test_row_0/C:col10/1731960753459/Put/seqid=0 2024-11-18T20:12:34,397 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35963 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-18T20:12:34,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8. 2024-11-18T20:12:34,398 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8. as already flushing 2024-11-18T20:12:34,398 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8. 2024-11-18T20:12:34,398 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 9d5f6ceb1ccdf5b797c44f37788d88e8, NAME => 'TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:12:34,398 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 9d5f6ceb1ccdf5b797c44f37788d88e8, NAME => 'TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:12:34,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9d5f6ceb1ccdf5b797c44f37788d88e8, NAME => 'TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9d5f6ceb1ccdf5b797c44f37788d88e8, NAME => 'TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:12:34,430 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073741987_1163 (size=12207) 2024-11-18T20:12:34,445 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073741986_1162 (size=48139) 2024-11-18T20:12:34,446 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=121, memsize=53.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/.tmp/A/82936db65f6b4993b25aa32eac0b21d7 2024-11-18T20:12:34,463 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/.tmp/B/676f1d9f947a49e98ed170a2cd6a1f95 is 50, key is test_row_0/B:col10/1731960753547/Put/seqid=0 2024-11-18T20:12:34,488 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073741988_1164 (size=12001) 2024-11-18T20:12:34,546 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:34,546 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:34,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47220 deadline: 1731960814543, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:34,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47158 deadline: 1731960814545, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:34,547 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:34,547 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:34,547 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:12:34,547 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:34,547 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:34,547 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:12:34,552 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35963 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-18T20:12:34,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8. 2024-11-18T20:12:34,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8. as already flushing 2024-11-18T20:12:34,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8. 2024-11-18T20:12:34,552 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 9d5f6ceb1ccdf5b797c44f37788d88e8, NAME => 'TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:12:34,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 9d5f6ceb1ccdf5b797c44f37788d88e8, NAME => 'TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:12:34,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9d5f6ceb1ccdf5b797c44f37788d88e8, NAME => 'TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9d5f6ceb1ccdf5b797c44f37788d88e8, NAME => 'TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:12:34,555 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:34,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47206 deadline: 1731960814553, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:34,555 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:34,555 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:34,555 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:12:34,568 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:34,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47184 deadline: 1731960814564, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:34,569 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:34,569 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:34,569 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:12:34,706 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35963 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-18T20:12:34,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8. 2024-11-18T20:12:34,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8. as already flushing 2024-11-18T20:12:34,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8. 2024-11-18T20:12:34,707 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 9d5f6ceb1ccdf5b797c44f37788d88e8, NAME => 'TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:12:34,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 9d5f6ceb1ccdf5b797c44f37788d88e8, NAME => 'TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:12:34,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9d5f6ceb1ccdf5b797c44f37788d88e8, NAME => 'TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9d5f6ceb1ccdf5b797c44f37788d88e8, NAME => 'TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:12:34,738 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9d5f6ceb1ccdf5b797c44f37788d88e8#A#compaction#137 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:12:34,739 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/.tmp/A/88bd77f166ab49d39e837bd064e2194d is 175, key is test_row_0/A:col10/1731960753459/Put/seqid=0 2024-11-18T20:12:34,781 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073741989_1165 (size=31161) 2024-11-18T20:12:34,809 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/.tmp/A/88bd77f166ab49d39e837bd064e2194d as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/A/88bd77f166ab49d39e837bd064e2194d 2024-11-18T20:12:34,822 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9d5f6ceb1ccdf5b797c44f37788d88e8/A of 9d5f6ceb1ccdf5b797c44f37788d88e8 into 88bd77f166ab49d39e837bd064e2194d(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:12:34,822 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9d5f6ceb1ccdf5b797c44f37788d88e8: 2024-11-18T20:12:34,822 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., storeName=9d5f6ceb1ccdf5b797c44f37788d88e8/A, priority=13, startTime=1731960754191; duration=0sec 2024-11-18T20:12:34,822 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:12:34,822 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9d5f6ceb1ccdf5b797c44f37788d88e8:A 2024-11-18T20:12:34,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-18T20:12:34,839 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/.tmp/C/e8c4015a7e704d1eb3191eef0cd574a9 as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/C/e8c4015a7e704d1eb3191eef0cd574a9 2024-11-18T20:12:34,848 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9d5f6ceb1ccdf5b797c44f37788d88e8/C of 9d5f6ceb1ccdf5b797c44f37788d88e8 into e8c4015a7e704d1eb3191eef0cd574a9(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:12:34,848 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9d5f6ceb1ccdf5b797c44f37788d88e8: 2024-11-18T20:12:34,848 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., storeName=9d5f6ceb1ccdf5b797c44f37788d88e8/C, priority=13, startTime=1731960754191; duration=0sec 2024-11-18T20:12:34,848 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:12:34,848 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9d5f6ceb1ccdf5b797c44f37788d88e8:C 2024-11-18T20:12:34,862 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35963 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-18T20:12:34,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8. 2024-11-18T20:12:34,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8. as already flushing 2024-11-18T20:12:34,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8. 2024-11-18T20:12:34,862 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 9d5f6ceb1ccdf5b797c44f37788d88e8, NAME => 'TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:12:34,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 9d5f6ceb1ccdf5b797c44f37788d88e8, NAME => 'TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:12:34,863 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:34,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47220 deadline: 1731960814854, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:34,863 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:34,863 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:34,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47182 deadline: 1731960814854, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:34,863 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:34,863 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:12:34,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9d5f6ceb1ccdf5b797c44f37788d88e8, NAME => 'TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9d5f6ceb1ccdf5b797c44f37788d88e8, NAME => 'TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:12:34,863 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:34,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47158 deadline: 1731960814855, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:34,864 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:34,864 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:34,864 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:12:34,864 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to c0a89b2656d4,35963,1731960718223 for 'test_row_2' in 9d5f6ceb1ccdf5b797c44f37788d88e8 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4241 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-18T20:12:34,865 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:34,865 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:34,865 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:12:34,866 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:34,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47206 deadline: 1731960814864, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:34,867 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:34,867 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:34,867 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:12:34,875 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:34,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47184 deadline: 1731960814875, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:34,876 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:34,876 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:34,876 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:12:34,889 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=121 (bloomFilter=true), to=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/.tmp/B/676f1d9f947a49e98ed170a2cd6a1f95 2024-11-18T20:12:34,915 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/.tmp/C/0dbae9b9d64c48abba6ceca4cca5a469 is 50, key is test_row_0/C:col10/1731960753547/Put/seqid=0 2024-11-18T20:12:34,974 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073741990_1166 (size=12001) 2024-11-18T20:12:35,018 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35963 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-18T20:12:35,019 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8. 2024-11-18T20:12:35,019 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8. as already flushing 2024-11-18T20:12:35,019 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8. 2024-11-18T20:12:35,019 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 9d5f6ceb1ccdf5b797c44f37788d88e8, NAME => 'TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:12:35,019 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 9d5f6ceb1ccdf5b797c44f37788d88e8, NAME => 'TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:12:35,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9d5f6ceb1ccdf5b797c44f37788d88e8, NAME => 'TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9d5f6ceb1ccdf5b797c44f37788d88e8, NAME => 'TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:12:35,173 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35963 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-18T20:12:35,173 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8. 2024-11-18T20:12:35,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8. as already flushing 2024-11-18T20:12:35,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8. 2024-11-18T20:12:35,174 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 9d5f6ceb1ccdf5b797c44f37788d88e8, NAME => 'TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:12:35,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 9d5f6ceb1ccdf5b797c44f37788d88e8, NAME => 'TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:12:35,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9d5f6ceb1ccdf5b797c44f37788d88e8, NAME => 'TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9d5f6ceb1ccdf5b797c44f37788d88e8, NAME => 'TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:12:35,327 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35963 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-18T20:12:35,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8. 2024-11-18T20:12:35,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8. as already flushing 2024-11-18T20:12:35,328 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8. 2024-11-18T20:12:35,328 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 9d5f6ceb1ccdf5b797c44f37788d88e8, NAME => 'TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:12:35,328 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 9d5f6ceb1ccdf5b797c44f37788d88e8, NAME => 'TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:12:35,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9d5f6ceb1ccdf5b797c44f37788d88e8, NAME => 'TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9d5f6ceb1ccdf5b797c44f37788d88e8, NAME => 'TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:12:35,374 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:35,374 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:35,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47158 deadline: 1731960815373, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:35,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47220 deadline: 1731960815373, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:35,374 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:35,374 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:35,374 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:12:35,375 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:35,376 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:35,376 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:12:35,376 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=121 (bloomFilter=true), to=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/.tmp/C/0dbae9b9d64c48abba6ceca4cca5a469 2024-11-18T20:12:35,377 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:35,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47206 deadline: 1731960815376, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:35,378 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:35,378 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:35,378 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:12:35,384 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:35,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47184 deadline: 1731960815384, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:35,385 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:35,385 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:35,385 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:12:35,391 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/.tmp/A/82936db65f6b4993b25aa32eac0b21d7 as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/A/82936db65f6b4993b25aa32eac0b21d7 2024-11-18T20:12:35,407 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/A/82936db65f6b4993b25aa32eac0b21d7, entries=250, sequenceid=121, filesize=47.0 K 2024-11-18T20:12:35,410 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/.tmp/B/676f1d9f947a49e98ed170a2cd6a1f95 as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/B/676f1d9f947a49e98ed170a2cd6a1f95 2024-11-18T20:12:35,422 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/B/676f1d9f947a49e98ed170a2cd6a1f95, entries=150, sequenceid=121, filesize=11.7 K 2024-11-18T20:12:35,427 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/.tmp/C/0dbae9b9d64c48abba6ceca4cca5a469 as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/C/0dbae9b9d64c48abba6ceca4cca5a469 2024-11-18T20:12:35,443 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/C/0dbae9b9d64c48abba6ceca4cca5a469, entries=150, sequenceid=121, filesize=11.7 K 2024-11-18T20:12:35,444 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~422.58 KB/432720, currentSize=40.25 KB/41220 for 9d5f6ceb1ccdf5b797c44f37788d88e8 in 1238ms, sequenceid=121, compaction requested=false 2024-11-18T20:12:35,444 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9d5f6ceb1ccdf5b797c44f37788d88e8: 2024-11-18T20:12:35,483 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35963 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-18T20:12:35,483 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8. 2024-11-18T20:12:35,483 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2902): Flushing 9d5f6ceb1ccdf5b797c44f37788d88e8 3/3 column families, dataSize=40.25 KB heapSize=106.22 KB 2024-11-18T20:12:35,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411183d4d9ff5f7e94c7f9b9268955608bf0e_9d5f6ceb1ccdf5b797c44f37788d88e8 is 50, key is test_row_0/A:col10/1731960754208/Put/seqid=0 2024-11-18T20:12:35,596 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073741991_1167 (size=12204) 2024-11-18T20:12:35,600 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:35,607 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411183d4d9ff5f7e94c7f9b9268955608bf0e_9d5f6ceb1ccdf5b797c44f37788d88e8 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411183d4d9ff5f7e94c7f9b9268955608bf0e_9d5f6ceb1ccdf5b797c44f37788d88e8 2024-11-18T20:12:35,609 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/.tmp/A/ff255c12130b43e9a829043f35525ddf, store: [table=TestAcidGuarantees family=A region=9d5f6ceb1ccdf5b797c44f37788d88e8] 2024-11-18T20:12:35,610 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/.tmp/A/ff255c12130b43e9a829043f35525ddf is 175, key is test_row_0/A:col10/1731960754208/Put/seqid=0 2024-11-18T20:12:35,661 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073741992_1168 (size=31005) 2024-11-18T20:12:35,662 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=133, memsize=13.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/.tmp/A/ff255c12130b43e9a829043f35525ddf 2024-11-18T20:12:35,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/.tmp/B/c43632556cca49a8951a5f3661364dfe is 50, key is test_row_0/B:col10/1731960754208/Put/seqid=0 2024-11-18T20:12:35,755 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073741993_1169 (size=12051) 2024-11-18T20:12:36,158 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=133 (bloomFilter=true), to=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/.tmp/B/c43632556cca49a8951a5f3661364dfe 2024-11-18T20:12:36,194 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/.tmp/C/ee7dc1a04dda47929cba204f921f23e7 is 50, key is test_row_0/C:col10/1731960754208/Put/seqid=0 2024-11-18T20:12:36,223 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073741994_1170 (size=12051) 2024-11-18T20:12:36,223 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=133 (bloomFilter=true), to=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/.tmp/C/ee7dc1a04dda47929cba204f921f23e7 2024-11-18T20:12:36,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/.tmp/A/ff255c12130b43e9a829043f35525ddf as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/A/ff255c12130b43e9a829043f35525ddf 2024-11-18T20:12:36,239 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/A/ff255c12130b43e9a829043f35525ddf, entries=150, sequenceid=133, filesize=30.3 K 2024-11-18T20:12:36,240 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/.tmp/B/c43632556cca49a8951a5f3661364dfe as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/B/c43632556cca49a8951a5f3661364dfe 2024-11-18T20:12:36,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,250 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/B/c43632556cca49a8951a5f3661364dfe, entries=150, sequenceid=133, filesize=11.8 K 2024-11-18T20:12:36,252 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/.tmp/C/ee7dc1a04dda47929cba204f921f23e7 as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/C/ee7dc1a04dda47929cba204f921f23e7 2024-11-18T20:12:36,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,266 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/C/ee7dc1a04dda47929cba204f921f23e7, entries=150, sequenceid=133, filesize=11.8 K 2024-11-18T20:12:36,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,269 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~106.17 KB/108720, currentSize=0 B/0 for 9d5f6ceb1ccdf5b797c44f37788d88e8 in 786ms, sequenceid=133, compaction requested=true 2024-11-18T20:12:36,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,269 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2603): Flush status journal for 9d5f6ceb1ccdf5b797c44f37788d88e8: 2024-11-18T20:12:36,269 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8. 2024-11-18T20:12:36,269 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=43 2024-11-18T20:12:36,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.HMaster(4169): Remote procedure done, pid=43 2024-11-18T20:12:36,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,274 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=43, resume processing ppid=42 2024-11-18T20:12:36,274 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=43, ppid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.5930 sec 2024-11-18T20:12:36,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,278 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees in 3.6000 sec 2024-11-18T20:12:36,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,533 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-18T20:12:36,534 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9d5f6ceb1ccdf5b797c44f37788d88e8 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-18T20:12:36,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(8855): Flush requested on 9d5f6ceb1ccdf5b797c44f37788d88e8 2024-11-18T20:12:36,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:36,573 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111843d874b4b648487d9fb850d89d68d699_9d5f6ceb1ccdf5b797c44f37788d88e8 is 50, key is test_row_0/A:col10/1731960756530/Put/seqid=0 2024-11-18T20:12:36,634 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:36,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47158 deadline: 1731960816625, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:36,634 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:36,634 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:36,634 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:36,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47184 deadline: 1731960816625, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:36,634 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:12:36,643 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073741995_1171 (size=24758) 2024-11-18T20:12:36,642 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:36,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47206 deadline: 1731960816628, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:36,643 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:36,643 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:36,643 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:12:36,645 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:36,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47220 deadline: 1731960816629, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:36,645 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:36,645 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:36,645 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:12:36,646 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:36,646 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:36,646 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:12:36,752 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:36,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47158 deadline: 1731960816743, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:36,754 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:36,754 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:36,754 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:12:36,755 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:36,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47206 deadline: 1731960816754, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:36,755 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:36,755 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:36,755 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:36,755 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:12:36,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47220 deadline: 1731960816755, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:36,756 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:36,756 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:36,756 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:36,756 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:12:36,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47184 deadline: 1731960816755, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:36,756 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:36,756 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:36,756 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:12:36,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-18T20:12:36,846 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-18T20:12:36,847 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-18T20:12:36,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] procedure2.ProcedureExecutor(1139): Stored pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees 2024-11-18T20:12:36,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-18T20:12:36,849 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-18T20:12:36,850 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-18T20:12:36,851 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=45, ppid=44, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-18T20:12:36,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-18T20:12:36,966 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:36,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47158 deadline: 1731960816964, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:36,966 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:36,966 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:36,966 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:36,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47206 deadline: 1731960816964, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:36,967 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:12:36,967 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:36,967 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:36,967 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:36,967 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:12:36,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47220 deadline: 1731960816964, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:36,967 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:36,967 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:36,967 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:12:36,968 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:36,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47184 deadline: 1731960816964, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:36,968 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:36,968 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:36,968 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:12:37,004 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35963 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-18T20:12:37,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8. 2024-11-18T20:12:37,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8. as already flushing 2024-11-18T20:12:37,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8. 2024-11-18T20:12:37,004 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 9d5f6ceb1ccdf5b797c44f37788d88e8, NAME => 'TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:12:37,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 9d5f6ceb1ccdf5b797c44f37788d88e8, NAME => 'TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:12:37,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9d5f6ceb1ccdf5b797c44f37788d88e8, NAME => 'TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9d5f6ceb1ccdf5b797c44f37788d88e8, NAME => 'TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:12:37,044 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:37,055 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111843d874b4b648487d9fb850d89d68d699_9d5f6ceb1ccdf5b797c44f37788d88e8 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111843d874b4b648487d9fb850d89d68d699_9d5f6ceb1ccdf5b797c44f37788d88e8 2024-11-18T20:12:37,056 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/.tmp/A/894d0df15ec64e6fa3be363e61ce27db, store: [table=TestAcidGuarantees family=A region=9d5f6ceb1ccdf5b797c44f37788d88e8] 2024-11-18T20:12:37,057 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/.tmp/A/894d0df15ec64e6fa3be363e61ce27db is 175, key is test_row_0/A:col10/1731960756530/Put/seqid=0 2024-11-18T20:12:37,060 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073741996_1172 (size=74395) 2024-11-18T20:12:37,061 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=144, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/.tmp/A/894d0df15ec64e6fa3be363e61ce27db 2024-11-18T20:12:37,078 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/.tmp/B/1f6395b0486c41b2abd10e52df1b1d2c is 50, key is test_row_0/B:col10/1731960756530/Put/seqid=0 2024-11-18T20:12:37,131 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073741997_1173 (size=12151) 2024-11-18T20:12:37,157 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35963 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-18T20:12:37,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8. 2024-11-18T20:12:37,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8. as already flushing 2024-11-18T20:12:37,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8. 2024-11-18T20:12:37,158 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 9d5f6ceb1ccdf5b797c44f37788d88e8, NAME => 'TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:12:37,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 9d5f6ceb1ccdf5b797c44f37788d88e8, NAME => 'TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:12:37,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9d5f6ceb1ccdf5b797c44f37788d88e8, NAME => 'TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9d5f6ceb1ccdf5b797c44f37788d88e8, NAME => 'TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:12:37,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-18T20:12:37,278 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:37,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47158 deadline: 1731960817274, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:37,279 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:37,279 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:37,279 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:12:37,279 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:37,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47184 deadline: 1731960817274, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:37,279 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:37,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47206 deadline: 1731960817274, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:37,280 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:37,280 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:37,280 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:37,280 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:37,280 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:37,280 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:12:37,280 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:12:37,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47220 deadline: 1731960817274, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:37,280 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:37,280 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:37,280 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:12:37,310 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35963 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-18T20:12:37,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8. 2024-11-18T20:12:37,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8. as already flushing 2024-11-18T20:12:37,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8. 2024-11-18T20:12:37,311 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 9d5f6ceb1ccdf5b797c44f37788d88e8, NAME => 'TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:12:37,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 9d5f6ceb1ccdf5b797c44f37788d88e8, NAME => 'TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:12:37,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9d5f6ceb1ccdf5b797c44f37788d88e8, NAME => 'TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9d5f6ceb1ccdf5b797c44f37788d88e8, NAME => 'TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:12:37,466 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35963 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-18T20:12:37,466 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8. 2024-11-18T20:12:37,466 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8. as already flushing 2024-11-18T20:12:37,466 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8. 2024-11-18T20:12:37,466 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 9d5f6ceb1ccdf5b797c44f37788d88e8, NAME => 'TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:12:37,467 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 9d5f6ceb1ccdf5b797c44f37788d88e8, NAME => 'TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:12:37,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9d5f6ceb1ccdf5b797c44f37788d88e8, NAME => 'TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9d5f6ceb1ccdf5b797c44f37788d88e8, NAME => 'TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:12:37,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-18T20:12:37,534 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=144 (bloomFilter=true), to=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/.tmp/B/1f6395b0486c41b2abd10e52df1b1d2c 2024-11-18T20:12:37,552 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/.tmp/C/ea1d95882f824602851c7f4e59a41594 is 50, key is test_row_0/C:col10/1731960756530/Put/seqid=0 2024-11-18T20:12:37,608 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073741998_1174 (size=12151) 2024-11-18T20:12:37,620 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35963 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-18T20:12:37,620 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8. 2024-11-18T20:12:37,620 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8. as already flushing 2024-11-18T20:12:37,620 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8. 2024-11-18T20:12:37,620 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 9d5f6ceb1ccdf5b797c44f37788d88e8, NAME => 'TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:12:37,620 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 9d5f6ceb1ccdf5b797c44f37788d88e8, NAME => 'TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:12:37,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9d5f6ceb1ccdf5b797c44f37788d88e8, NAME => 'TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9d5f6ceb1ccdf5b797c44f37788d88e8, NAME => 'TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:12:37,775 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35963 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-18T20:12:37,775 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8. 2024-11-18T20:12:37,775 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8. as already flushing 2024-11-18T20:12:37,775 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8. 2024-11-18T20:12:37,775 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 9d5f6ceb1ccdf5b797c44f37788d88e8, NAME => 'TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:12:37,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 9d5f6ceb1ccdf5b797c44f37788d88e8, NAME => 'TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:12:37,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9d5f6ceb1ccdf5b797c44f37788d88e8, NAME => 'TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9d5f6ceb1ccdf5b797c44f37788d88e8, NAME => 'TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:12:37,789 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:37,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47206 deadline: 1731960817783, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:37,790 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:37,790 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:37,790 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:12:37,797 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:37,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47220 deadline: 1731960817794, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:37,797 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:37,797 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:37,797 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:37,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47184 deadline: 1731960817794, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:37,797 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:12:37,798 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:37,798 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:37,798 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:12:37,803 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:37,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47158 deadline: 1731960817799, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:37,804 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:37,804 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:37,804 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:12:37,922 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-18T20:12:37,922 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-11-18T20:12:37,932 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35963 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-18T20:12:37,932 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8. 2024-11-18T20:12:37,932 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8. as already flushing 2024-11-18T20:12:37,932 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8. 2024-11-18T20:12:37,932 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 9d5f6ceb1ccdf5b797c44f37788d88e8, NAME => 'TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:12:37,932 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 9d5f6ceb1ccdf5b797c44f37788d88e8, NAME => 'TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:12:37,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9d5f6ceb1ccdf5b797c44f37788d88e8, NAME => 'TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9d5f6ceb1ccdf5b797c44f37788d88e8, NAME => 'TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:12:37,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-18T20:12:38,008 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=144 (bloomFilter=true), to=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/.tmp/C/ea1d95882f824602851c7f4e59a41594 2024-11-18T20:12:38,016 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/.tmp/A/894d0df15ec64e6fa3be363e61ce27db as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/A/894d0df15ec64e6fa3be363e61ce27db 2024-11-18T20:12:38,037 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/A/894d0df15ec64e6fa3be363e61ce27db, entries=400, sequenceid=144, filesize=72.7 K 2024-11-18T20:12:38,039 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/.tmp/B/1f6395b0486c41b2abd10e52df1b1d2c as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/B/1f6395b0486c41b2abd10e52df1b1d2c 2024-11-18T20:12:38,048 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/B/1f6395b0486c41b2abd10e52df1b1d2c, entries=150, sequenceid=144, filesize=11.9 K 2024-11-18T20:12:38,052 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/.tmp/C/ea1d95882f824602851c7f4e59a41594 as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/C/ea1d95882f824602851c7f4e59a41594 2024-11-18T20:12:38,069 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/C/ea1d95882f824602851c7f4e59a41594, entries=150, sequenceid=144, filesize=11.9 K 2024-11-18T20:12:38,070 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=154.31 KB/158010 for 9d5f6ceb1ccdf5b797c44f37788d88e8 in 1537ms, sequenceid=144, compaction requested=true 2024-11-18T20:12:38,070 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9d5f6ceb1ccdf5b797c44f37788d88e8: 2024-11-18T20:12:38,071 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-18T20:12:38,071 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9d5f6ceb1ccdf5b797c44f37788d88e8:A, priority=-2147483648, current under compaction store size is 1 2024-11-18T20:12:38,071 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:12:38,071 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-18T20:12:38,072 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9d5f6ceb1ccdf5b797c44f37788d88e8:B, priority=-2147483648, current under compaction store size is 2 2024-11-18T20:12:38,072 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:12:38,072 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9d5f6ceb1ccdf5b797c44f37788d88e8:C, priority=-2147483648, current under compaction store size is 3 2024-11-18T20:12:38,072 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T20:12:38,075 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 184700 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-18T20:12:38,075 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48410 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-18T20:12:38,075 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HStore(1541): 9d5f6ceb1ccdf5b797c44f37788d88e8/A is initiating minor compaction (all files) 2024-11-18T20:12:38,075 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HStore(1541): 9d5f6ceb1ccdf5b797c44f37788d88e8/B is initiating minor compaction (all files) 2024-11-18T20:12:38,075 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9d5f6ceb1ccdf5b797c44f37788d88e8/A in TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8. 2024-11-18T20:12:38,075 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9d5f6ceb1ccdf5b797c44f37788d88e8/B in TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8. 2024-11-18T20:12:38,075 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/A/88bd77f166ab49d39e837bd064e2194d, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/A/82936db65f6b4993b25aa32eac0b21d7, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/A/ff255c12130b43e9a829043f35525ddf, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/A/894d0df15ec64e6fa3be363e61ce27db] into tmpdir=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/.tmp, totalSize=180.4 K 2024-11-18T20:12:38,075 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/B/105a322f94b241759e166e95d61e1a9b, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/B/676f1d9f947a49e98ed170a2cd6a1f95, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/B/c43632556cca49a8951a5f3661364dfe, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/B/1f6395b0486c41b2abd10e52df1b1d2c] into tmpdir=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/.tmp, totalSize=47.3 K 2024-11-18T20:12:38,075 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8. 2024-11-18T20:12:38,075 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8. files: [hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/A/88bd77f166ab49d39e837bd064e2194d, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/A/82936db65f6b4993b25aa32eac0b21d7, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/A/ff255c12130b43e9a829043f35525ddf, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/A/894d0df15ec64e6fa3be363e61ce27db] 2024-11-18T20:12:38,076 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.Compactor(225): Compacting 105a322f94b241759e166e95d61e1a9b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1731960752824 2024-11-18T20:12:38,076 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] compactions.Compactor(225): Compacting 88bd77f166ab49d39e837bd064e2194d, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1731960752824 2024-11-18T20:12:38,077 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.Compactor(225): Compacting 676f1d9f947a49e98ed170a2cd6a1f95, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=121, earliestPutTs=1731960753541 2024-11-18T20:12:38,077 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] compactions.Compactor(225): Compacting 82936db65f6b4993b25aa32eac0b21d7, keycount=250, bloomtype=ROW, size=47.0 K, encoding=NONE, compression=NONE, seqNum=121, earliestPutTs=1731960753541 2024-11-18T20:12:38,077 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] compactions.Compactor(225): Compacting ff255c12130b43e9a829043f35525ddf, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1731960754208 2024-11-18T20:12:38,077 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.Compactor(225): Compacting c43632556cca49a8951a5f3661364dfe, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1731960754208 2024-11-18T20:12:38,078 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] compactions.Compactor(225): Compacting 894d0df15ec64e6fa3be363e61ce27db, keycount=400, bloomtype=ROW, size=72.7 K, encoding=NONE, compression=NONE, seqNum=144, earliestPutTs=1731960756463 2024-11-18T20:12:38,078 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.Compactor(225): Compacting 1f6395b0486c41b2abd10e52df1b1d2c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=144, earliestPutTs=1731960756526 2024-11-18T20:12:38,085 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35963 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-18T20:12:38,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8. 2024-11-18T20:12:38,085 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2902): Flushing 9d5f6ceb1ccdf5b797c44f37788d88e8 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-18T20:12:38,102 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9d5f6ceb1ccdf5b797c44f37788d88e8#B#compaction#149 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:12:38,103 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/.tmp/B/8d2fa5da60c54e37802fccd9d8178247 is 50, key is test_row_0/B:col10/1731960756530/Put/seqid=0 2024-11-18T20:12:38,126 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=9d5f6ceb1ccdf5b797c44f37788d88e8] 2024-11-18T20:12:38,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411184e9c6b4033d5409d88759cf9ffbfa09b_9d5f6ceb1ccdf5b797c44f37788d88e8 is 50, key is test_row_0/A:col10/1731960756623/Put/seqid=0 2024-11-18T20:12:38,166 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241118ecfe72e688d9450cb90f737aaf9bfb33_9d5f6ceb1ccdf5b797c44f37788d88e8 store=[table=TestAcidGuarantees family=A region=9d5f6ceb1ccdf5b797c44f37788d88e8] 2024-11-18T20:12:38,171 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241118ecfe72e688d9450cb90f737aaf9bfb33_9d5f6ceb1ccdf5b797c44f37788d88e8, store=[table=TestAcidGuarantees family=A region=9d5f6ceb1ccdf5b797c44f37788d88e8] 2024-11-18T20:12:38,172 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118ecfe72e688d9450cb90f737aaf9bfb33_9d5f6ceb1ccdf5b797c44f37788d88e8 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=9d5f6ceb1ccdf5b797c44f37788d88e8] 2024-11-18T20:12:38,196 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073741999_1175 (size=12493) 2024-11-18T20:12:38,248 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742000_1176 (size=12304) 2024-11-18T20:12:38,249 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:38,256 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411184e9c6b4033d5409d88759cf9ffbfa09b_9d5f6ceb1ccdf5b797c44f37788d88e8 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411184e9c6b4033d5409d88759cf9ffbfa09b_9d5f6ceb1ccdf5b797c44f37788d88e8 2024-11-18T20:12:38,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/.tmp/A/787ada51f532477fb87d941fb3d151bd, store: [table=TestAcidGuarantees family=A region=9d5f6ceb1ccdf5b797c44f37788d88e8] 2024-11-18T20:12:38,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/.tmp/A/787ada51f532477fb87d941fb3d151bd is 175, key is test_row_0/A:col10/1731960756623/Put/seqid=0 2024-11-18T20:12:38,271 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742001_1177 (size=4469) 2024-11-18T20:12:38,297 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742002_1178 (size=31105) 2024-11-18T20:12:38,308 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=170, memsize=51.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/.tmp/A/787ada51f532477fb87d941fb3d151bd 2024-11-18T20:12:38,321 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/.tmp/B/56ce60d2bd7842ab84770ce977f26de4 is 50, key is test_row_0/B:col10/1731960756623/Put/seqid=0 2024-11-18T20:12:38,352 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742003_1179 (size=12151) 2024-11-18T20:12:38,609 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/.tmp/B/8d2fa5da60c54e37802fccd9d8178247 as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/B/8d2fa5da60c54e37802fccd9d8178247 2024-11-18T20:12:38,625 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 9d5f6ceb1ccdf5b797c44f37788d88e8/B of 9d5f6ceb1ccdf5b797c44f37788d88e8 into 8d2fa5da60c54e37802fccd9d8178247(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:12:38,625 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9d5f6ceb1ccdf5b797c44f37788d88e8: 2024-11-18T20:12:38,625 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., storeName=9d5f6ceb1ccdf5b797c44f37788d88e8/B, priority=12, startTime=1731960758071; duration=0sec 2024-11-18T20:12:38,626 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T20:12:38,626 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9d5f6ceb1ccdf5b797c44f37788d88e8:B 2024-11-18T20:12:38,626 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-18T20:12:38,629 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48410 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-18T20:12:38,629 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HStore(1541): 9d5f6ceb1ccdf5b797c44f37788d88e8/C is initiating minor compaction (all files) 2024-11-18T20:12:38,629 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9d5f6ceb1ccdf5b797c44f37788d88e8/C in TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8. 2024-11-18T20:12:38,629 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/C/e8c4015a7e704d1eb3191eef0cd574a9, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/C/0dbae9b9d64c48abba6ceca4cca5a469, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/C/ee7dc1a04dda47929cba204f921f23e7, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/C/ea1d95882f824602851c7f4e59a41594] into tmpdir=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/.tmp, totalSize=47.3 K 2024-11-18T20:12:38,629 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.Compactor(225): Compacting e8c4015a7e704d1eb3191eef0cd574a9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1731960752824 2024-11-18T20:12:38,630 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.Compactor(225): Compacting 0dbae9b9d64c48abba6ceca4cca5a469, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=121, earliestPutTs=1731960753541 2024-11-18T20:12:38,630 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.Compactor(225): Compacting ee7dc1a04dda47929cba204f921f23e7, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1731960754208 2024-11-18T20:12:38,631 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.Compactor(225): Compacting ea1d95882f824602851c7f4e59a41594, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=144, earliestPutTs=1731960756526 2024-11-18T20:12:38,659 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9d5f6ceb1ccdf5b797c44f37788d88e8#C#compaction#153 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:12:38,659 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/.tmp/C/89b3d2b5737841fbaf37bdb21fdcd126 is 50, key is test_row_0/C:col10/1731960756530/Put/seqid=0 2024-11-18T20:12:38,671 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742004_1180 (size=12493) 2024-11-18T20:12:38,672 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9d5f6ceb1ccdf5b797c44f37788d88e8#A#compaction#150 average throughput is 0.04 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:12:38,673 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/.tmp/A/5ae222a3e3734d61933f7895b0296150 is 175, key is test_row_0/A:col10/1731960756530/Put/seqid=0 2024-11-18T20:12:38,682 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742005_1181 (size=31447) 2024-11-18T20:12:38,754 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=170 (bloomFilter=true), to=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/.tmp/B/56ce60d2bd7842ab84770ce977f26de4 2024-11-18T20:12:38,763 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/.tmp/C/affa94e1672a44af8bd19a93bcc96f16 is 50, key is test_row_0/C:col10/1731960756623/Put/seqid=0 2024-11-18T20:12:38,784 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742006_1182 (size=12151) 2024-11-18T20:12:38,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(8855): Flush requested on 9d5f6ceb1ccdf5b797c44f37788d88e8 2024-11-18T20:12:38,806 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8. as already flushing 2024-11-18T20:12:38,816 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:38,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47158 deadline: 1731960818814, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:38,817 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to c0a89b2656d4,35963,1731960718223 for 'test_row_0' in 9d5f6ceb1ccdf5b797c44f37788d88e8 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2192 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-18T20:12:38,818 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:38,818 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:38,818 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:12:38,819 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:38,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47220 deadline: 1731960818814, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:38,820 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:38,820 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:38,820 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:38,820 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:12:38,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47184 deadline: 1731960818815, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:38,822 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:38,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47206 deadline: 1731960818816, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:38,823 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:38,823 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:38,823 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:12:38,829 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:38,829 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:38,829 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:12:38,885 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:38,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47182 deadline: 1731960818885, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:38,886 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to c0a89b2656d4,35963,1731960718223 for 'test_row_2' in 9d5f6ceb1ccdf5b797c44f37788d88e8 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8262 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-18T20:12:38,887 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:38,887 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:38,887 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:12:38,926 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:38,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47220 deadline: 1731960818923, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:38,926 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:38,926 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:38,927 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:12:38,937 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:38,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47184 deadline: 1731960818934, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:38,938 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:38,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47206 deadline: 1731960818935, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:38,938 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:38,938 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:38,938 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:12:38,938 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:38,938 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:38,938 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:12:38,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-18T20:12:39,083 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/.tmp/C/89b3d2b5737841fbaf37bdb21fdcd126 as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/C/89b3d2b5737841fbaf37bdb21fdcd126 2024-11-18T20:12:39,090 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/.tmp/A/5ae222a3e3734d61933f7895b0296150 as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/A/5ae222a3e3734d61933f7895b0296150 2024-11-18T20:12:39,094 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 9d5f6ceb1ccdf5b797c44f37788d88e8/C of 9d5f6ceb1ccdf5b797c44f37788d88e8 into 89b3d2b5737841fbaf37bdb21fdcd126(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:12:39,094 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9d5f6ceb1ccdf5b797c44f37788d88e8: 2024-11-18T20:12:39,094 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., storeName=9d5f6ceb1ccdf5b797c44f37788d88e8/C, priority=12, startTime=1731960758072; duration=0sec 2024-11-18T20:12:39,094 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:12:39,094 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9d5f6ceb1ccdf5b797c44f37788d88e8:C 2024-11-18T20:12:39,099 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 9d5f6ceb1ccdf5b797c44f37788d88e8/A of 9d5f6ceb1ccdf5b797c44f37788d88e8 into 5ae222a3e3734d61933f7895b0296150(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 1sec to execute. 2024-11-18T20:12:39,099 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9d5f6ceb1ccdf5b797c44f37788d88e8: 2024-11-18T20:12:39,099 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., storeName=9d5f6ceb1ccdf5b797c44f37788d88e8/A, priority=12, startTime=1731960758070; duration=1sec 2024-11-18T20:12:39,100 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:12:39,100 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9d5f6ceb1ccdf5b797c44f37788d88e8:A 2024-11-18T20:12:39,136 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:39,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47220 deadline: 1731960819134, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:39,137 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:39,137 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:39,137 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:12:39,143 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:39,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47184 deadline: 1731960819143, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:39,144 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:39,144 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:39,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47206 deadline: 1731960819143, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:39,144 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:39,144 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:12:39,153 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:39,153 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:39,153 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:12:39,185 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=170 (bloomFilter=true), to=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/.tmp/C/affa94e1672a44af8bd19a93bcc96f16 2024-11-18T20:12:39,194 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/.tmp/A/787ada51f532477fb87d941fb3d151bd as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/A/787ada51f532477fb87d941fb3d151bd 2024-11-18T20:12:39,198 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/A/787ada51f532477fb87d941fb3d151bd, entries=150, sequenceid=170, filesize=30.4 K 2024-11-18T20:12:39,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/.tmp/B/56ce60d2bd7842ab84770ce977f26de4 as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/B/56ce60d2bd7842ab84770ce977f26de4 2024-11-18T20:12:39,211 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/B/56ce60d2bd7842ab84770ce977f26de4, entries=150, sequenceid=170, filesize=11.9 K 2024-11-18T20:12:39,212 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/.tmp/C/affa94e1672a44af8bd19a93bcc96f16 as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/C/affa94e1672a44af8bd19a93bcc96f16 2024-11-18T20:12:39,220 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/C/affa94e1672a44af8bd19a93bcc96f16, entries=150, sequenceid=170, filesize=11.9 K 2024-11-18T20:12:39,221 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for 9d5f6ceb1ccdf5b797c44f37788d88e8 in 1136ms, sequenceid=170, compaction requested=false 2024-11-18T20:12:39,221 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2603): Flush status journal for 9d5f6ceb1ccdf5b797c44f37788d88e8: 2024-11-18T20:12:39,221 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8. 2024-11-18T20:12:39,221 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=45 2024-11-18T20:12:39,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.HMaster(4169): Remote procedure done, pid=45 2024-11-18T20:12:39,224 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=45, resume processing ppid=44 2024-11-18T20:12:39,224 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=45, ppid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3710 sec 2024-11-18T20:12:39,226 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees in 2.3780 sec 2024-11-18T20:12:39,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(8855): Flush requested on 9d5f6ceb1ccdf5b797c44f37788d88e8 2024-11-18T20:12:39,446 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-18T20:12:39,447 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9d5f6ceb1ccdf5b797c44f37788d88e8 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-18T20:12:39,469 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118fb1706e21572435b8df5edd2e6929de1_9d5f6ceb1ccdf5b797c44f37788d88e8 is 50, key is test_row_0/A:col10/1731960759445/Put/seqid=0 2024-11-18T20:12:39,492 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742007_1183 (size=12304) 2024-11-18T20:12:39,547 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:39,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47220 deadline: 1731960819542, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:39,548 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:39,548 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:39,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47206 deadline: 1731960819543, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:39,548 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:39,548 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:12:39,548 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:39,548 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:39,548 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:39,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47184 deadline: 1731960819544, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:39,548 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:12:39,548 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:39,548 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:39,549 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:12:39,656 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:39,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47220 deadline: 1731960819653, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:39,656 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:39,656 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:39,656 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:12:39,656 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:39,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47206 deadline: 1731960819653, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:39,657 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:39,657 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:39,657 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:12:39,657 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:39,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47184 deadline: 1731960819653, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:39,657 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:39,657 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:39,657 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:12:39,865 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:39,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47220 deadline: 1731960819864, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:39,865 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:39,865 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:39,866 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:12:39,865 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:39,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47184 deadline: 1731960819864, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:39,866 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:39,866 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:39,866 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:39,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47206 deadline: 1731960819864, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:39,866 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:12:39,866 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:39,866 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:39,866 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:12:39,893 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:39,907 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118fb1706e21572435b8df5edd2e6929de1_9d5f6ceb1ccdf5b797c44f37788d88e8 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118fb1706e21572435b8df5edd2e6929de1_9d5f6ceb1ccdf5b797c44f37788d88e8 2024-11-18T20:12:39,909 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/.tmp/A/ad0bd1618e0e4e7d80a473e735edbb71, store: [table=TestAcidGuarantees family=A region=9d5f6ceb1ccdf5b797c44f37788d88e8] 2024-11-18T20:12:39,909 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/.tmp/A/ad0bd1618e0e4e7d80a473e735edbb71 is 175, key is test_row_0/A:col10/1731960759445/Put/seqid=0 2024-11-18T20:12:39,935 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742008_1184 (size=31105) 2024-11-18T20:12:39,936 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=184, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/.tmp/A/ad0bd1618e0e4e7d80a473e735edbb71 2024-11-18T20:12:39,961 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/.tmp/B/d528bbee6dbd422f859d63caa5f4f4cc is 50, key is test_row_0/B:col10/1731960759445/Put/seqid=0 2024-11-18T20:12:39,986 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742009_1185 (size=12151) 2024-11-18T20:12:40,175 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:40,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47184 deadline: 1731960820173, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:40,176 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:40,176 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:40,176 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:12:40,182 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:40,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47220 deadline: 1731960820173, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:40,185 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:40,185 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:40,185 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:12:40,186 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:40,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47206 deadline: 1731960820173, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:40,189 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:40,189 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:40,189 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:12:40,389 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=184 (bloomFilter=true), to=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/.tmp/B/d528bbee6dbd422f859d63caa5f4f4cc 2024-11-18T20:12:40,398 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/.tmp/C/1314a6e353784c598c758b6208c319dc is 50, key is test_row_0/C:col10/1731960759445/Put/seqid=0 2024-11-18T20:12:40,406 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742010_1186 (size=12151) 2024-11-18T20:12:40,684 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:40,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47184 deadline: 1731960820683, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:40,685 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:40,685 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:40,685 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:12:40,695 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:40,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47220 deadline: 1731960820693, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:40,695 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:40,695 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:40,695 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:12:40,705 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:40,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47206 deadline: 1731960820703, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:40,706 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:40,706 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:40,706 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:12:40,807 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=184 (bloomFilter=true), to=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/.tmp/C/1314a6e353784c598c758b6208c319dc 2024-11-18T20:12:40,813 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/.tmp/A/ad0bd1618e0e4e7d80a473e735edbb71 as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/A/ad0bd1618e0e4e7d80a473e735edbb71 2024-11-18T20:12:40,817 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/A/ad0bd1618e0e4e7d80a473e735edbb71, entries=150, sequenceid=184, filesize=30.4 K 2024-11-18T20:12:40,819 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/.tmp/B/d528bbee6dbd422f859d63caa5f4f4cc as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/B/d528bbee6dbd422f859d63caa5f4f4cc 2024-11-18T20:12:40,823 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/B/d528bbee6dbd422f859d63caa5f4f4cc, entries=150, sequenceid=184, filesize=11.9 K 2024-11-18T20:12:40,824 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/.tmp/C/1314a6e353784c598c758b6208c319dc as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/C/1314a6e353784c598c758b6208c319dc 2024-11-18T20:12:40,829 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/C/1314a6e353784c598c758b6208c319dc, entries=150, sequenceid=184, filesize=11.9 K 2024-11-18T20:12:40,830 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=154.31 KB/158010 for 9d5f6ceb1ccdf5b797c44f37788d88e8 in 1384ms, sequenceid=184, compaction requested=true 2024-11-18T20:12:40,830 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9d5f6ceb1ccdf5b797c44f37788d88e8: 2024-11-18T20:12:40,830 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9d5f6ceb1ccdf5b797c44f37788d88e8:A, priority=-2147483648, current under compaction store size is 1 2024-11-18T20:12:40,830 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:12:40,830 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T20:12:40,830 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9d5f6ceb1ccdf5b797c44f37788d88e8:B, priority=-2147483648, current under compaction store size is 2 2024-11-18T20:12:40,830 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:12:40,830 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T20:12:40,831 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9d5f6ceb1ccdf5b797c44f37788d88e8:C, priority=-2147483648, current under compaction store size is 3 2024-11-18T20:12:40,831 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T20:12:40,832 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 93657 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T20:12:40,832 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T20:12:40,832 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HStore(1541): 9d5f6ceb1ccdf5b797c44f37788d88e8/A is initiating minor compaction (all files) 2024-11-18T20:12:40,832 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HStore(1541): 9d5f6ceb1ccdf5b797c44f37788d88e8/B is initiating minor compaction (all files) 2024-11-18T20:12:40,832 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9d5f6ceb1ccdf5b797c44f37788d88e8/A in TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8. 2024-11-18T20:12:40,832 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9d5f6ceb1ccdf5b797c44f37788d88e8/B in TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8. 2024-11-18T20:12:40,832 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/A/5ae222a3e3734d61933f7895b0296150, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/A/787ada51f532477fb87d941fb3d151bd, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/A/ad0bd1618e0e4e7d80a473e735edbb71] into tmpdir=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/.tmp, totalSize=91.5 K 2024-11-18T20:12:40,832 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/B/8d2fa5da60c54e37802fccd9d8178247, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/B/56ce60d2bd7842ab84770ce977f26de4, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/B/d528bbee6dbd422f859d63caa5f4f4cc] into tmpdir=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/.tmp, totalSize=35.9 K 2024-11-18T20:12:40,832 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8. 2024-11-18T20:12:40,832 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8. files: [hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/A/5ae222a3e3734d61933f7895b0296150, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/A/787ada51f532477fb87d941fb3d151bd, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/A/ad0bd1618e0e4e7d80a473e735edbb71] 2024-11-18T20:12:40,832 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.Compactor(225): Compacting 8d2fa5da60c54e37802fccd9d8178247, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=144, earliestPutTs=1731960756526 2024-11-18T20:12:40,833 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5ae222a3e3734d61933f7895b0296150, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=144, earliestPutTs=1731960756526 2024-11-18T20:12:40,834 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.Compactor(225): Compacting 56ce60d2bd7842ab84770ce977f26de4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1731960756561 2024-11-18T20:12:40,834 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.Compactor(225): Compacting d528bbee6dbd422f859d63caa5f4f4cc, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=184, earliestPutTs=1731960758810 2024-11-18T20:12:40,834 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] compactions.Compactor(225): Compacting 787ada51f532477fb87d941fb3d151bd, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1731960756561 2024-11-18T20:12:40,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(8855): Flush requested on 9d5f6ceb1ccdf5b797c44f37788d88e8 2024-11-18T20:12:40,835 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-18T20:12:40,835 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] compactions.Compactor(225): Compacting ad0bd1618e0e4e7d80a473e735edbb71, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=184, earliestPutTs=1731960758810 2024-11-18T20:12:40,835 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9d5f6ceb1ccdf5b797c44f37788d88e8 3/3 column families, dataSize=161.02 KB heapSize=422.63 KB 2024-11-18T20:12:40,851 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9d5f6ceb1ccdf5b797c44f37788d88e8#B#compaction#158 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:12:40,852 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/.tmp/B/ad154585b3934afcabd208d580fba23a is 50, key is test_row_0/B:col10/1731960759445/Put/seqid=0 2024-11-18T20:12:40,861 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=9d5f6ceb1ccdf5b797c44f37788d88e8] 2024-11-18T20:12:40,867 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:40,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47158 deadline: 1731960820865, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:40,867 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:40,868 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:40,868 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:12:40,874 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241118aec1c1e2cfa24c2992201f08dfcfba8d_9d5f6ceb1ccdf5b797c44f37788d88e8 store=[table=TestAcidGuarantees family=A region=9d5f6ceb1ccdf5b797c44f37788d88e8] 2024-11-18T20:12:40,875 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241118aec1c1e2cfa24c2992201f08dfcfba8d_9d5f6ceb1ccdf5b797c44f37788d88e8, store=[table=TestAcidGuarantees family=A region=9d5f6ceb1ccdf5b797c44f37788d88e8] 2024-11-18T20:12:40,875 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118aec1c1e2cfa24c2992201f08dfcfba8d_9d5f6ceb1ccdf5b797c44f37788d88e8 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=9d5f6ceb1ccdf5b797c44f37788d88e8] 2024-11-18T20:12:40,878 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118b0a3195e45464f848230dde07269f7b8_9d5f6ceb1ccdf5b797c44f37788d88e8 is 50, key is test_row_0/A:col10/1731960760833/Put/seqid=0 2024-11-18T20:12:40,887 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742011_1187 (size=12595) 2024-11-18T20:12:40,889 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742013_1189 (size=12304) 2024-11-18T20:12:40,915 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742012_1188 (size=4469) 2024-11-18T20:12:40,978 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:40,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47158 deadline: 1731960820973, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:40,978 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:40,978 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:40,978 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:12:41,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-18T20:12:41,004 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-18T20:12:41,004 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-18T20:12:41,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] procedure2.ProcedureExecutor(1139): Stored pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees 2024-11-18T20:12:41,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-18T20:12:41,006 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-18T20:12:41,006 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-18T20:12:41,006 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=47, ppid=46, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-18T20:12:41,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-18T20:12:41,157 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35963 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-18T20:12:41,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8. 2024-11-18T20:12:41,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8. as already flushing 2024-11-18T20:12:41,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8. 2024-11-18T20:12:41,158 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 9d5f6ceb1ccdf5b797c44f37788d88e8, NAME => 'TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:12:41,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 9d5f6ceb1ccdf5b797c44f37788d88e8, NAME => 'TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:12:41,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9d5f6ceb1ccdf5b797c44f37788d88e8, NAME => 'TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9d5f6ceb1ccdf5b797c44f37788d88e8, NAME => 'TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:12:41,186 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:41,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47158 deadline: 1731960821183, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:41,187 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:41,187 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:41,187 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:12:41,289 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:41,294 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/.tmp/B/ad154585b3934afcabd208d580fba23a as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/B/ad154585b3934afcabd208d580fba23a 2024-11-18T20:12:41,296 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118b0a3195e45464f848230dde07269f7b8_9d5f6ceb1ccdf5b797c44f37788d88e8 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118b0a3195e45464f848230dde07269f7b8_9d5f6ceb1ccdf5b797c44f37788d88e8 2024-11-18T20:12:41,297 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/.tmp/A/7a393fd61b4a4e4a9a5f1450d4d17879, store: [table=TestAcidGuarantees family=A region=9d5f6ceb1ccdf5b797c44f37788d88e8] 2024-11-18T20:12:41,297 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/.tmp/A/7a393fd61b4a4e4a9a5f1450d4d17879 is 175, key is test_row_0/A:col10/1731960760833/Put/seqid=0 2024-11-18T20:12:41,300 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9d5f6ceb1ccdf5b797c44f37788d88e8/B of 9d5f6ceb1ccdf5b797c44f37788d88e8 into ad154585b3934afcabd208d580fba23a(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:12:41,300 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9d5f6ceb1ccdf5b797c44f37788d88e8: 2024-11-18T20:12:41,300 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., storeName=9d5f6ceb1ccdf5b797c44f37788d88e8/B, priority=13, startTime=1731960760830; duration=0sec 2024-11-18T20:12:41,300 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T20:12:41,300 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9d5f6ceb1ccdf5b797c44f37788d88e8:B 2024-11-18T20:12:41,300 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T20:12:41,302 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T20:12:41,302 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HStore(1541): 9d5f6ceb1ccdf5b797c44f37788d88e8/C is initiating minor compaction (all files) 2024-11-18T20:12:41,302 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9d5f6ceb1ccdf5b797c44f37788d88e8/C in TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8. 2024-11-18T20:12:41,302 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/C/89b3d2b5737841fbaf37bdb21fdcd126, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/C/affa94e1672a44af8bd19a93bcc96f16, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/C/1314a6e353784c598c758b6208c319dc] into tmpdir=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/.tmp, totalSize=35.9 K 2024-11-18T20:12:41,303 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.Compactor(225): Compacting 89b3d2b5737841fbaf37bdb21fdcd126, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=144, earliestPutTs=1731960756526 2024-11-18T20:12:41,303 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.Compactor(225): Compacting affa94e1672a44af8bd19a93bcc96f16, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1731960756561 2024-11-18T20:12:41,303 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742014_1190 (size=31105) 2024-11-18T20:12:41,304 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.Compactor(225): Compacting 1314a6e353784c598c758b6208c319dc, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=184, earliestPutTs=1731960758810 2024-11-18T20:12:41,304 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=211, memsize=53.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/.tmp/A/7a393fd61b4a4e4a9a5f1450d4d17879 2024-11-18T20:12:41,311 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35963 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-18T20:12:41,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8. 2024-11-18T20:12:41,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8. as already flushing 2024-11-18T20:12:41,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8. 2024-11-18T20:12:41,311 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 9d5f6ceb1ccdf5b797c44f37788d88e8, NAME => 'TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:12:41,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 9d5f6ceb1ccdf5b797c44f37788d88e8, NAME => 'TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:12:41,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9d5f6ceb1ccdf5b797c44f37788d88e8, NAME => 'TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9d5f6ceb1ccdf5b797c44f37788d88e8, NAME => 'TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:12:41,316 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9d5f6ceb1ccdf5b797c44f37788d88e8#A#compaction#159 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:12:41,317 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/.tmp/A/95dba009994343528996c305712ce70d is 175, key is test_row_0/A:col10/1731960759445/Put/seqid=0 2024-11-18T20:12:41,323 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9d5f6ceb1ccdf5b797c44f37788d88e8#C#compaction#161 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:12:41,324 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/.tmp/C/927ee34d8c3e431c942bde6ee59d18ec is 50, key is test_row_0/C:col10/1731960759445/Put/seqid=0 2024-11-18T20:12:41,331 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/.tmp/B/1b11a567514544a6b0a4cde5e2c37dcf is 50, key is test_row_0/B:col10/1731960760833/Put/seqid=0 2024-11-18T20:12:41,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-18T20:12:41,335 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742016_1192 (size=12595) 2024-11-18T20:12:41,343 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742015_1191 (size=31549) 2024-11-18T20:12:41,350 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742017_1193 (size=12151) 2024-11-18T20:12:41,351 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=211 (bloomFilter=true), to=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/.tmp/B/1b11a567514544a6b0a4cde5e2c37dcf 2024-11-18T20:12:41,351 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/.tmp/A/95dba009994343528996c305712ce70d as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/A/95dba009994343528996c305712ce70d 2024-11-18T20:12:41,358 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9d5f6ceb1ccdf5b797c44f37788d88e8/A of 9d5f6ceb1ccdf5b797c44f37788d88e8 into 95dba009994343528996c305712ce70d(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:12:41,358 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9d5f6ceb1ccdf5b797c44f37788d88e8: 2024-11-18T20:12:41,358 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., storeName=9d5f6ceb1ccdf5b797c44f37788d88e8/A, priority=13, startTime=1731960760830; duration=0sec 2024-11-18T20:12:41,358 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:12:41,358 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9d5f6ceb1ccdf5b797c44f37788d88e8:A 2024-11-18T20:12:41,369 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/.tmp/C/374ee136231143aeb4b9fc08372ec6cf is 50, key is test_row_0/C:col10/1731960760833/Put/seqid=0 2024-11-18T20:12:41,387 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742018_1194 (size=12151) 2024-11-18T20:12:41,463 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35963 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-18T20:12:41,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8. 2024-11-18T20:12:41,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8. as already flushing 2024-11-18T20:12:41,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8. 2024-11-18T20:12:41,464 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 9d5f6ceb1ccdf5b797c44f37788d88e8, NAME => 'TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:12:41,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 9d5f6ceb1ccdf5b797c44f37788d88e8, NAME => 'TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:12:41,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9d5f6ceb1ccdf5b797c44f37788d88e8, NAME => 'TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9d5f6ceb1ccdf5b797c44f37788d88e8, NAME => 'TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:12:41,494 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:41,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47158 deadline: 1731960821493, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:41,494 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:41,494 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:41,494 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:12:41,616 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35963 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-18T20:12:41,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8. 2024-11-18T20:12:41,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8. as already flushing 2024-11-18T20:12:41,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8. 2024-11-18T20:12:41,617 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 9d5f6ceb1ccdf5b797c44f37788d88e8, NAME => 'TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:12:41,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 9d5f6ceb1ccdf5b797c44f37788d88e8, NAME => 'TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:12:41,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9d5f6ceb1ccdf5b797c44f37788d88e8, NAME => 'TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9d5f6ceb1ccdf5b797c44f37788d88e8, NAME => 'TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:12:41,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-18T20:12:41,693 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:41,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47184 deadline: 1731960821693, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:41,694 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to c0a89b2656d4,35963,1731960718223 for 'test_row_0' in 9d5f6ceb1ccdf5b797c44f37788d88e8 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2150 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-18T20:12:41,694 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:41,695 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:41,695 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:12:41,704 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:41,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47220 deadline: 1731960821704, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:41,705 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to c0a89b2656d4,35963,1731960718223 for 'test_row_1' in 9d5f6ceb1ccdf5b797c44f37788d88e8 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2163 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-18T20:12:41,706 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:41,706 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:41,706 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:12:41,719 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:41,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47206 deadline: 1731960821715, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:41,720 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to c0a89b2656d4,35963,1731960718223 for 'test_row_0' in 9d5f6ceb1ccdf5b797c44f37788d88e8 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2177 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-18T20:12:41,721 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:41,721 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:41,721 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:12:41,744 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/.tmp/C/927ee34d8c3e431c942bde6ee59d18ec as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/C/927ee34d8c3e431c942bde6ee59d18ec 2024-11-18T20:12:41,762 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9d5f6ceb1ccdf5b797c44f37788d88e8/C of 9d5f6ceb1ccdf5b797c44f37788d88e8 into 927ee34d8c3e431c942bde6ee59d18ec(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:12:41,762 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9d5f6ceb1ccdf5b797c44f37788d88e8: 2024-11-18T20:12:41,762 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., storeName=9d5f6ceb1ccdf5b797c44f37788d88e8/C, priority=13, startTime=1731960760830; duration=0sec 2024-11-18T20:12:41,763 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:12:41,763 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9d5f6ceb1ccdf5b797c44f37788d88e8:C 2024-11-18T20:12:41,770 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35963 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-18T20:12:41,770 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8. 2024-11-18T20:12:41,770 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8. as already flushing 2024-11-18T20:12:41,771 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8. 2024-11-18T20:12:41,771 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 9d5f6ceb1ccdf5b797c44f37788d88e8, NAME => 'TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:12:41,771 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 9d5f6ceb1ccdf5b797c44f37788d88e8, NAME => 'TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:12:41,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9d5f6ceb1ccdf5b797c44f37788d88e8, NAME => 'TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9d5f6ceb1ccdf5b797c44f37788d88e8, NAME => 'TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:12:41,788 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=211 (bloomFilter=true), to=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/.tmp/C/374ee136231143aeb4b9fc08372ec6cf 2024-11-18T20:12:41,797 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/.tmp/A/7a393fd61b4a4e4a9a5f1450d4d17879 as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/A/7a393fd61b4a4e4a9a5f1450d4d17879 2024-11-18T20:12:41,811 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/A/7a393fd61b4a4e4a9a5f1450d4d17879, entries=150, sequenceid=211, filesize=30.4 K 2024-11-18T20:12:41,814 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/.tmp/B/1b11a567514544a6b0a4cde5e2c37dcf as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/B/1b11a567514544a6b0a4cde5e2c37dcf 2024-11-18T20:12:41,821 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/B/1b11a567514544a6b0a4cde5e2c37dcf, entries=150, sequenceid=211, filesize=11.9 K 2024-11-18T20:12:41,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:41,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:41,829 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/.tmp/C/374ee136231143aeb4b9fc08372ec6cf as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/C/374ee136231143aeb4b9fc08372ec6cf 2024-11-18T20:12:41,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:41,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:41,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:41,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:41,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:41,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:41,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:41,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:41,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:41,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:41,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:41,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:41,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:41,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:41,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:41,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:41,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:41,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:41,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:41,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:41,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:41,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:41,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:41,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:41,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:41,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:41,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:41,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:41,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:41,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:41,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:41,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:41,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:41,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:41,850 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/C/374ee136231143aeb4b9fc08372ec6cf, entries=150, sequenceid=211, filesize=11.9 K 2024-11-18T20:12:41,851 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~422.58 KB/432720, currentSize=40.25 KB/41220 for 9d5f6ceb1ccdf5b797c44f37788d88e8 in 1015ms, sequenceid=211, compaction requested=false 2024-11-18T20:12:41,851 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9d5f6ceb1ccdf5b797c44f37788d88e8: 2024-11-18T20:12:41,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:41,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:41,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:41,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:41,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:41,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:41,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:41,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:41,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:41,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:41,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:41,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:41,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:41,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:41,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:41,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:41,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:41,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:41,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:41,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:41,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:41,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:41,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:41,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:41,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:41,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:41,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:41,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:41,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:41,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:41,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:41,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:41,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:41,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:41,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:41,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:41,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:41,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:41,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:41,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:41,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:41,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:41,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:41,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:41,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:41,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:41,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:41,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:41,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:41,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:41,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:41,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:41,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:41,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:41,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:41,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:41,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:41,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:41,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:41,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:41,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:41,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:41,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:41,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:41,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:41,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:41,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:41,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:41,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:41,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:41,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:41,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:41,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:41,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:41,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:41,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:41,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:41,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:41,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:41,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:41,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:41,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:41,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:41,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:41,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:41,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:41,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:41,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:41,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:41,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:41,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:41,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:41,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:41,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:41,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:41,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:41,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:41,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:41,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:41,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:41,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:41,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:41,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:41,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:41,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:41,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:41,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:41,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:41,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:41,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:41,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:41,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:41,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:41,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:41,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:41,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:41,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:41,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:41,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:41,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:41,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:41,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:41,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:41,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:41,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:41,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:41,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:41,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:41,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:41,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:41,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:41,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:41,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:41,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:41,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:41,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:41,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:41,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:41,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:41,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:41,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:41,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:41,924 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35963 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-18T20:12:41,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8. 2024-11-18T20:12:41,925 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2902): Flushing 9d5f6ceb1ccdf5b797c44f37788d88e8 3/3 column families, dataSize=40.25 KB heapSize=106.22 KB 2024-11-18T20:12:41,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:41,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:41,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:41,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:41,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:41,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:41,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:41,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:41,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:41,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:41,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:41,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:41,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:41,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:41,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:41,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:41,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:41,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:41,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:41,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:41,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:41,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411181fb7c2e57ab142aab585e683013e1eb1_9d5f6ceb1ccdf5b797c44f37788d88e8 is 50, key is test_row_0/A:col10/1731960760843/Put/seqid=0 2024-11-18T20:12:41,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:41,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:41,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:41,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:41,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:41,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:41,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:41,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:41,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:41,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:41,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:41,955 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742019_1195 (size=9814) 2024-11-18T20:12:41,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:41,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:41,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:41,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:41,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:41,962 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411181fb7c2e57ab142aab585e683013e1eb1_9d5f6ceb1ccdf5b797c44f37788d88e8 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411181fb7c2e57ab142aab585e683013e1eb1_9d5f6ceb1ccdf5b797c44f37788d88e8 2024-11-18T20:12:41,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:41,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:41,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/.tmp/A/89e2d4925fd9481f952e2920f34c6983, store: [table=TestAcidGuarantees family=A region=9d5f6ceb1ccdf5b797c44f37788d88e8] 2024-11-18T20:12:41,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:41,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/.tmp/A/89e2d4925fd9481f952e2920f34c6983 is 175, key is test_row_0/A:col10/1731960760843/Put/seqid=0 2024-11-18T20:12:41,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:41,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:41,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:41,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:41,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:41,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:41,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:41,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:41,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:41,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:41,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:41,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:41,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:41,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:41,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:41,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:41,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:41,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:41,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:41,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:41,978 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742020_1196 (size=22461) 2024-11-18T20:12:41,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:41,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:41,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:41,979 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=223, memsize=13.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/.tmp/A/89e2d4925fd9481f952e2920f34c6983 2024-11-18T20:12:41,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:41,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:41,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:41,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:41,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:41,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:41,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:41,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:41,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:41,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:41,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:41,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:41,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:41,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:41,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:41,990 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/.tmp/B/fab2f392ddfa4526a1505dc8b8fee5dd is 50, key is test_row_0/B:col10/1731960760843/Put/seqid=0 2024-11-18T20:12:41,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:41,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:41,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:41,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:41,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:41,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:41,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:41,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:41,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:41,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:41,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:41,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:41,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:41,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:41,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:41,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:42,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:42,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:42,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:42,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:42,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:42,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:42,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:42,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:42,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:42,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:42,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:42,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:42,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:42,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:42,013 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742021_1197 (size=9757) 2024-11-18T20:12:42,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:42,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:42,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:42,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:42,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:42,017 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=223 (bloomFilter=true), to=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/.tmp/B/fab2f392ddfa4526a1505dc8b8fee5dd 2024-11-18T20:12:42,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:42,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:42,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:42,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(8855): Flush requested on 9d5f6ceb1ccdf5b797c44f37788d88e8 2024-11-18T20:12:42,019 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8. as already flushing 2024-11-18T20:12:42,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:42,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:42,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:42,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:42,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:42,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:42,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:42,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:42,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:42,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:42,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:42,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:42,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:42,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:42,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:42,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:42,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:42,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:42,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:42,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:42,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:42,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:42,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:42,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:42,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:42,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:42,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:42,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:42,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:42,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:42,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:42,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:42,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:42,035 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/.tmp/C/97e5c5217b46428483dc1346f30b84b4 is 50, key is test_row_0/C:col10/1731960760843/Put/seqid=0 2024-11-18T20:12:42,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:42,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:42,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:42,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:42,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:42,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:42,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:42,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:42,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:42,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:42,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:42,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:42,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:42,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:42,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:42,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:42,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:42,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:42,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:42,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:42,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:42,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:42,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:42,070 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742022_1198 (size=9757) 2024-11-18T20:12:42,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:42,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-18T20:12:42,196 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:42,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47158 deadline: 1731960822194, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:42,197 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:42,197 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:42,197 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:12:42,305 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:42,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47158 deadline: 1731960822303, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:42,306 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:42,306 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:42,306 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:12:42,403 INFO [master/c0a89b2656d4:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-11-18T20:12:42,403 INFO [master/c0a89b2656d4:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-11-18T20:12:42,471 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=223 (bloomFilter=true), to=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/.tmp/C/97e5c5217b46428483dc1346f30b84b4 2024-11-18T20:12:42,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/.tmp/A/89e2d4925fd9481f952e2920f34c6983 as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/A/89e2d4925fd9481f952e2920f34c6983 2024-11-18T20:12:42,481 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/A/89e2d4925fd9481f952e2920f34c6983, entries=100, sequenceid=223, filesize=21.9 K 2024-11-18T20:12:42,483 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/.tmp/B/fab2f392ddfa4526a1505dc8b8fee5dd as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/B/fab2f392ddfa4526a1505dc8b8fee5dd 2024-11-18T20:12:42,488 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/B/fab2f392ddfa4526a1505dc8b8fee5dd, entries=100, sequenceid=223, filesize=9.5 K 2024-11-18T20:12:42,489 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/.tmp/C/97e5c5217b46428483dc1346f30b84b4 as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/C/97e5c5217b46428483dc1346f30b84b4 2024-11-18T20:12:42,493 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/C/97e5c5217b46428483dc1346f30b84b4, entries=100, sequenceid=223, filesize=9.5 K 2024-11-18T20:12:42,494 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~106.17 KB/108720, currentSize=161.02 KB/164880 for 9d5f6ceb1ccdf5b797c44f37788d88e8 in 570ms, sequenceid=223, compaction requested=true 2024-11-18T20:12:42,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2603): Flush status journal for 9d5f6ceb1ccdf5b797c44f37788d88e8: 2024-11-18T20:12:42,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8. 2024-11-18T20:12:42,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=47 2024-11-18T20:12:42,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.HMaster(4169): Remote procedure done, pid=47 2024-11-18T20:12:42,498 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=47, resume processing ppid=46 2024-11-18T20:12:42,498 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=47, ppid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4890 sec 2024-11-18T20:12:42,500 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees in 1.4950 sec 2024-11-18T20:12:42,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(8855): Flush requested on 9d5f6ceb1ccdf5b797c44f37788d88e8 2024-11-18T20:12:42,516 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-18T20:12:42,516 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9d5f6ceb1ccdf5b797c44f37788d88e8 3/3 column families, dataSize=167.72 KB heapSize=440.20 KB 2024-11-18T20:12:42,528 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111840b857ea9a5648e688c9784e57a8e2f4_9d5f6ceb1ccdf5b797c44f37788d88e8 is 50, key is test_row_0/A:col10/1731960762515/Put/seqid=0 2024-11-18T20:12:42,538 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:42,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47158 deadline: 1731960822536, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:42,539 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:42,539 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:42,539 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:12:42,543 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742023_1199 (size=12304) 2024-11-18T20:12:42,644 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:42,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47158 deadline: 1731960822643, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:42,644 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:42,644 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:42,644 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:12:42,853 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:42,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47158 deadline: 1731960822853, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:42,854 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:42,854 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:42,854 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:12:42,943 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:42,947 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111840b857ea9a5648e688c9784e57a8e2f4_9d5f6ceb1ccdf5b797c44f37788d88e8 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111840b857ea9a5648e688c9784e57a8e2f4_9d5f6ceb1ccdf5b797c44f37788d88e8 2024-11-18T20:12:42,948 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/.tmp/A/e7b2ff26190f486c85a8871d7ebd7b66, store: [table=TestAcidGuarantees family=A region=9d5f6ceb1ccdf5b797c44f37788d88e8] 2024-11-18T20:12:42,949 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/.tmp/A/e7b2ff26190f486c85a8871d7ebd7b66 is 175, key is test_row_0/A:col10/1731960762515/Put/seqid=0 2024-11-18T20:12:42,952 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742024_1200 (size=31105) 2024-11-18T20:12:42,954 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=251, memsize=55.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/.tmp/A/e7b2ff26190f486c85a8871d7ebd7b66 2024-11-18T20:12:42,962 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/.tmp/B/7a083120e0a64294921239d966d2e2bb is 50, key is test_row_0/B:col10/1731960762515/Put/seqid=0 2024-11-18T20:12:42,965 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742025_1201 (size=12151) 2024-11-18T20:12:43,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-18T20:12:43,164 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-18T20:12:43,164 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-18T20:12:43,164 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:43,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47158 deadline: 1731960823163, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:43,165 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:43,165 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:43,165 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:12:43,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] procedure2.ProcedureExecutor(1139): Stored pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees 2024-11-18T20:12:43,166 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-18T20:12:43,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-18T20:12:43,166 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-18T20:12:43,167 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=49, ppid=48, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-18T20:12:43,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-18T20:12:43,318 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35963 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-18T20:12:43,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8. 2024-11-18T20:12:43,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8. as already flushing 2024-11-18T20:12:43,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8. 2024-11-18T20:12:43,319 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 9d5f6ceb1ccdf5b797c44f37788d88e8, NAME => 'TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:12:43,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 9d5f6ceb1ccdf5b797c44f37788d88e8, NAME => 'TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:12:43,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9d5f6ceb1ccdf5b797c44f37788d88e8, NAME => 'TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9d5f6ceb1ccdf5b797c44f37788d88e8, NAME => 'TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:12:43,366 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=251 (bloomFilter=true), to=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/.tmp/B/7a083120e0a64294921239d966d2e2bb 2024-11-18T20:12:43,374 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/.tmp/C/8ae46bb8bf3b474588d6716479bf416c is 50, key is test_row_0/C:col10/1731960762515/Put/seqid=0 2024-11-18T20:12:43,378 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742026_1202 (size=12151) 2024-11-18T20:12:43,471 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35963 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-18T20:12:43,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8. 2024-11-18T20:12:43,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8. as already flushing 2024-11-18T20:12:43,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8. 2024-11-18T20:12:43,472 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 9d5f6ceb1ccdf5b797c44f37788d88e8, NAME => 'TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:12:43,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 9d5f6ceb1ccdf5b797c44f37788d88e8, NAME => 'TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:12:43,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9d5f6ceb1ccdf5b797c44f37788d88e8, NAME => 'TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9d5f6ceb1ccdf5b797c44f37788d88e8, NAME => 'TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:12:43,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-18T20:12:43,624 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35963 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-18T20:12:43,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8. 2024-11-18T20:12:43,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8. as already flushing 2024-11-18T20:12:43,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8. 2024-11-18T20:12:43,624 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 9d5f6ceb1ccdf5b797c44f37788d88e8, NAME => 'TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:12:43,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 9d5f6ceb1ccdf5b797c44f37788d88e8, NAME => 'TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:12:43,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9d5f6ceb1ccdf5b797c44f37788d88e8, NAME => 'TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9d5f6ceb1ccdf5b797c44f37788d88e8, NAME => 'TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:12:43,674 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:43,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47158 deadline: 1731960823673, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:43,675 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:43,675 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:43,675 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:12:43,723 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:43,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47184 deadline: 1731960823723, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:43,724 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:43,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47220 deadline: 1731960823723, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:43,724 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to c0a89b2656d4,35963,1731960718223 for 'test_row_0' in 9d5f6ceb1ccdf5b797c44f37788d88e8 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4181 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-18T20:12:43,724 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to c0a89b2656d4,35963,1731960718223 for 'test_row_1' in 9d5f6ceb1ccdf5b797c44f37788d88e8 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4182 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-18T20:12:43,724 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:43,725 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:43,725 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:43,725 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:43,725 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:12:43,725 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:12:43,743 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:43,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47206 deadline: 1731960823743, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:43,744 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to c0a89b2656d4,35963,1731960718223 for 'test_row_0' in 9d5f6ceb1ccdf5b797c44f37788d88e8 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4200 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-18T20:12:43,744 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:43,744 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:43,744 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:12:43,777 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35963 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-18T20:12:43,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8. 2024-11-18T20:12:43,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8. as already flushing 2024-11-18T20:12:43,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8. 2024-11-18T20:12:43,777 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 9d5f6ceb1ccdf5b797c44f37788d88e8, NAME => 'TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:12:43,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 9d5f6ceb1ccdf5b797c44f37788d88e8, NAME => 'TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:12:43,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9d5f6ceb1ccdf5b797c44f37788d88e8, NAME => 'TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9d5f6ceb1ccdf5b797c44f37788d88e8, NAME => 'TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:12:43,778 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=251 (bloomFilter=true), to=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/.tmp/C/8ae46bb8bf3b474588d6716479bf416c 2024-11-18T20:12:43,784 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/.tmp/A/e7b2ff26190f486c85a8871d7ebd7b66 as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/A/e7b2ff26190f486c85a8871d7ebd7b66 2024-11-18T20:12:43,788 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/A/e7b2ff26190f486c85a8871d7ebd7b66, entries=150, sequenceid=251, filesize=30.4 K 2024-11-18T20:12:43,789 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/.tmp/B/7a083120e0a64294921239d966d2e2bb as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/B/7a083120e0a64294921239d966d2e2bb 2024-11-18T20:12:43,792 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/B/7a083120e0a64294921239d966d2e2bb, entries=150, sequenceid=251, filesize=11.9 K 2024-11-18T20:12:43,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-18T20:12:43,793 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/.tmp/C/8ae46bb8bf3b474588d6716479bf416c as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/C/8ae46bb8bf3b474588d6716479bf416c 2024-11-18T20:12:43,798 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/C/8ae46bb8bf3b474588d6716479bf416c, entries=150, sequenceid=251, filesize=11.9 K 2024-11-18T20:12:43,798 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~167.72 KB/171750, heapSize ~440.16 KB/450720, currentSize=33.54 KB/34350 for 9d5f6ceb1ccdf5b797c44f37788d88e8 in 1282ms, sequenceid=251, compaction requested=true 2024-11-18T20:12:43,798 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9d5f6ceb1ccdf5b797c44f37788d88e8: 2024-11-18T20:12:43,799 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9d5f6ceb1ccdf5b797c44f37788d88e8:A, priority=-2147483648, current under compaction store size is 1 2024-11-18T20:12:43,799 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-18T20:12:43,799 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:12:43,799 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9d5f6ceb1ccdf5b797c44f37788d88e8:B, priority=-2147483648, current under compaction store size is 2 2024-11-18T20:12:43,799 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:12:43,799 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-18T20:12:43,799 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9d5f6ceb1ccdf5b797c44f37788d88e8:C, priority=-2147483648, current under compaction store size is 3 2024-11-18T20:12:43,799 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T20:12:43,800 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 116220 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-18T20:12:43,800 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 46654 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-18T20:12:43,800 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HStore(1541): 9d5f6ceb1ccdf5b797c44f37788d88e8/A is initiating minor compaction (all files) 2024-11-18T20:12:43,800 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HStore(1541): 9d5f6ceb1ccdf5b797c44f37788d88e8/B is initiating minor compaction (all files) 2024-11-18T20:12:43,800 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9d5f6ceb1ccdf5b797c44f37788d88e8/A in TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8. 2024-11-18T20:12:43,800 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9d5f6ceb1ccdf5b797c44f37788d88e8/B in TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8. 2024-11-18T20:12:43,800 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/A/95dba009994343528996c305712ce70d, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/A/7a393fd61b4a4e4a9a5f1450d4d17879, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/A/89e2d4925fd9481f952e2920f34c6983, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/A/e7b2ff26190f486c85a8871d7ebd7b66] into tmpdir=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/.tmp, totalSize=113.5 K 2024-11-18T20:12:43,800 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8. 2024-11-18T20:12:43,800 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/B/ad154585b3934afcabd208d580fba23a, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/B/1b11a567514544a6b0a4cde5e2c37dcf, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/B/fab2f392ddfa4526a1505dc8b8fee5dd, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/B/7a083120e0a64294921239d966d2e2bb] into tmpdir=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/.tmp, totalSize=45.6 K 2024-11-18T20:12:43,800 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8. files: [hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/A/95dba009994343528996c305712ce70d, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/A/7a393fd61b4a4e4a9a5f1450d4d17879, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/A/89e2d4925fd9481f952e2920f34c6983, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/A/e7b2ff26190f486c85a8871d7ebd7b66] 2024-11-18T20:12:43,801 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.Compactor(225): Compacting ad154585b3934afcabd208d580fba23a, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=184, earliestPutTs=1731960758810 2024-11-18T20:12:43,801 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] compactions.Compactor(225): Compacting 95dba009994343528996c305712ce70d, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=184, earliestPutTs=1731960758810 2024-11-18T20:12:43,801 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.Compactor(225): Compacting 1b11a567514544a6b0a4cde5e2c37dcf, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=211, earliestPutTs=1731960759532 2024-11-18T20:12:43,801 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7a393fd61b4a4e4a9a5f1450d4d17879, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=211, earliestPutTs=1731960759532 2024-11-18T20:12:43,804 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] compactions.Compactor(225): Compacting 89e2d4925fd9481f952e2920f34c6983, keycount=100, bloomtype=ROW, size=21.9 K, encoding=NONE, compression=NONE, seqNum=223, earliestPutTs=1731960760843 2024-11-18T20:12:43,804 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.Compactor(225): Compacting fab2f392ddfa4526a1505dc8b8fee5dd, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=223, earliestPutTs=1731960760843 2024-11-18T20:12:43,805 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] compactions.Compactor(225): Compacting e7b2ff26190f486c85a8871d7ebd7b66, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=251, earliestPutTs=1731960762158 2024-11-18T20:12:43,805 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.Compactor(225): Compacting 7a083120e0a64294921239d966d2e2bb, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=251, earliestPutTs=1731960762158 2024-11-18T20:12:43,813 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=9d5f6ceb1ccdf5b797c44f37788d88e8] 2024-11-18T20:12:43,815 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9d5f6ceb1ccdf5b797c44f37788d88e8#B#compaction#171 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:12:43,816 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241118096ab11e0a6c46d9a73ebf0efd3025a6_9d5f6ceb1ccdf5b797c44f37788d88e8 store=[table=TestAcidGuarantees family=A region=9d5f6ceb1ccdf5b797c44f37788d88e8] 2024-11-18T20:12:43,816 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/.tmp/B/800fa60bdfe242acbe36cb89ebbc3e5a is 50, key is test_row_0/B:col10/1731960762515/Put/seqid=0 2024-11-18T20:12:43,818 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241118096ab11e0a6c46d9a73ebf0efd3025a6_9d5f6ceb1ccdf5b797c44f37788d88e8, store=[table=TestAcidGuarantees family=A region=9d5f6ceb1ccdf5b797c44f37788d88e8] 2024-11-18T20:12:43,818 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118096ab11e0a6c46d9a73ebf0efd3025a6_9d5f6ceb1ccdf5b797c44f37788d88e8 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=9d5f6ceb1ccdf5b797c44f37788d88e8] 2024-11-18T20:12:43,824 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742027_1203 (size=12731) 2024-11-18T20:12:43,829 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/.tmp/B/800fa60bdfe242acbe36cb89ebbc3e5a as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/B/800fa60bdfe242acbe36cb89ebbc3e5a 2024-11-18T20:12:43,836 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 9d5f6ceb1ccdf5b797c44f37788d88e8/B of 9d5f6ceb1ccdf5b797c44f37788d88e8 into 800fa60bdfe242acbe36cb89ebbc3e5a(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:12:43,836 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9d5f6ceb1ccdf5b797c44f37788d88e8: 2024-11-18T20:12:43,836 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., storeName=9d5f6ceb1ccdf5b797c44f37788d88e8/B, priority=12, startTime=1731960763799; duration=0sec 2024-11-18T20:12:43,836 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T20:12:43,836 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9d5f6ceb1ccdf5b797c44f37788d88e8:B 2024-11-18T20:12:43,836 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-18T20:12:43,838 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 46654 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-18T20:12:43,838 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HStore(1541): 9d5f6ceb1ccdf5b797c44f37788d88e8/C is initiating minor compaction (all files) 2024-11-18T20:12:43,838 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9d5f6ceb1ccdf5b797c44f37788d88e8/C in TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8. 2024-11-18T20:12:43,838 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/C/927ee34d8c3e431c942bde6ee59d18ec, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/C/374ee136231143aeb4b9fc08372ec6cf, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/C/97e5c5217b46428483dc1346f30b84b4, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/C/8ae46bb8bf3b474588d6716479bf416c] into tmpdir=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/.tmp, totalSize=45.6 K 2024-11-18T20:12:43,839 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.Compactor(225): Compacting 927ee34d8c3e431c942bde6ee59d18ec, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=184, earliestPutTs=1731960758810 2024-11-18T20:12:43,839 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.Compactor(225): Compacting 374ee136231143aeb4b9fc08372ec6cf, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=211, earliestPutTs=1731960759532 2024-11-18T20:12:43,839 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742028_1204 (size=4469) 2024-11-18T20:12:43,840 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.Compactor(225): Compacting 97e5c5217b46428483dc1346f30b84b4, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=223, earliestPutTs=1731960760843 2024-11-18T20:12:43,840 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.Compactor(225): Compacting 8ae46bb8bf3b474588d6716479bf416c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=251, earliestPutTs=1731960762158 2024-11-18T20:12:43,842 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9d5f6ceb1ccdf5b797c44f37788d88e8#A#compaction#170 average throughput is 0.84 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:12:43,842 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/.tmp/A/d17bc84cdc69479dbea8e28c1714df64 is 175, key is test_row_0/A:col10/1731960762515/Put/seqid=0 2024-11-18T20:12:43,850 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742029_1205 (size=31685) 2024-11-18T20:12:43,856 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9d5f6ceb1ccdf5b797c44f37788d88e8#C#compaction#172 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:12:43,857 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/.tmp/C/7d0afbc55b954b81a3dd35686040e709 is 50, key is test_row_0/C:col10/1731960762515/Put/seqid=0 2024-11-18T20:12:43,858 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/.tmp/A/d17bc84cdc69479dbea8e28c1714df64 as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/A/d17bc84cdc69479dbea8e28c1714df64 2024-11-18T20:12:43,865 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 9d5f6ceb1ccdf5b797c44f37788d88e8/A of 9d5f6ceb1ccdf5b797c44f37788d88e8 into d17bc84cdc69479dbea8e28c1714df64(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:12:43,865 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9d5f6ceb1ccdf5b797c44f37788d88e8: 2024-11-18T20:12:43,865 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., storeName=9d5f6ceb1ccdf5b797c44f37788d88e8/A, priority=12, startTime=1731960763799; duration=0sec 2024-11-18T20:12:43,865 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:12:43,865 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9d5f6ceb1ccdf5b797c44f37788d88e8:A 2024-11-18T20:12:43,872 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742030_1206 (size=12731) 2024-11-18T20:12:43,879 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/.tmp/C/7d0afbc55b954b81a3dd35686040e709 as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/C/7d0afbc55b954b81a3dd35686040e709 2024-11-18T20:12:43,889 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 9d5f6ceb1ccdf5b797c44f37788d88e8/C of 9d5f6ceb1ccdf5b797c44f37788d88e8 into 7d0afbc55b954b81a3dd35686040e709(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:12:43,889 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9d5f6ceb1ccdf5b797c44f37788d88e8: 2024-11-18T20:12:43,889 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., storeName=9d5f6ceb1ccdf5b797c44f37788d88e8/C, priority=12, startTime=1731960763799; duration=0sec 2024-11-18T20:12:43,889 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:12:43,889 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9d5f6ceb1ccdf5b797c44f37788d88e8:C 2024-11-18T20:12:43,929 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35963 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-18T20:12:43,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8. 2024-11-18T20:12:43,930 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2902): Flushing 9d5f6ceb1ccdf5b797c44f37788d88e8 3/3 column families, dataSize=33.54 KB heapSize=88.64 KB 2024-11-18T20:12:43,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118fcd9a6c5a19f46ef8039b75339201f30_9d5f6ceb1ccdf5b797c44f37788d88e8 is 50, key is test_row_0/A:col10/1731960762534/Put/seqid=0 2024-11-18T20:12:43,960 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742031_1207 (size=12404) 2024-11-18T20:12:44,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-18T20:12:44,361 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:44,367 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118fcd9a6c5a19f46ef8039b75339201f30_9d5f6ceb1ccdf5b797c44f37788d88e8 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118fcd9a6c5a19f46ef8039b75339201f30_9d5f6ceb1ccdf5b797c44f37788d88e8 2024-11-18T20:12:44,367 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/.tmp/A/079e3ea2fd8b40739e0a8f155a1c7204, store: [table=TestAcidGuarantees family=A region=9d5f6ceb1ccdf5b797c44f37788d88e8] 2024-11-18T20:12:44,368 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/.tmp/A/079e3ea2fd8b40739e0a8f155a1c7204 is 175, key is test_row_0/A:col10/1731960762534/Put/seqid=0 2024-11-18T20:12:44,371 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742032_1208 (size=31205) 2024-11-18T20:12:44,698 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8. as already flushing 2024-11-18T20:12:44,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(8855): Flush requested on 9d5f6ceb1ccdf5b797c44f37788d88e8 2024-11-18T20:12:44,764 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:44,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47158 deadline: 1731960824761, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:44,764 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:44,764 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:44,764 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:12:44,772 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=262, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/.tmp/A/079e3ea2fd8b40739e0a8f155a1c7204 2024-11-18T20:12:44,781 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/.tmp/B/a68d44c905d6476e83be80eda3b1f65f is 50, key is test_row_0/B:col10/1731960762534/Put/seqid=0 2024-11-18T20:12:44,785 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742033_1209 (size=12251) 2024-11-18T20:12:44,874 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:44,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47158 deadline: 1731960824873, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:44,875 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:44,875 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:44,875 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:12:45,084 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:45,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47158 deadline: 1731960825083, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:45,084 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:45,084 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:45,084 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:12:45,186 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=262 (bloomFilter=true), to=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/.tmp/B/a68d44c905d6476e83be80eda3b1f65f 2024-11-18T20:12:45,192 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/.tmp/C/d2bbddc6235345a69a5b3754bdae8661 is 50, key is test_row_0/C:col10/1731960762534/Put/seqid=0 2024-11-18T20:12:45,196 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742034_1210 (size=12251) 2024-11-18T20:12:45,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-18T20:12:45,394 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:45,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47158 deadline: 1731960825393, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:45,394 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:45,394 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:45,394 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:12:45,596 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=262 (bloomFilter=true), to=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/.tmp/C/d2bbddc6235345a69a5b3754bdae8661 2024-11-18T20:12:45,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/.tmp/A/079e3ea2fd8b40739e0a8f155a1c7204 as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/A/079e3ea2fd8b40739e0a8f155a1c7204 2024-11-18T20:12:45,606 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/A/079e3ea2fd8b40739e0a8f155a1c7204, entries=150, sequenceid=262, filesize=30.5 K 2024-11-18T20:12:45,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/.tmp/B/a68d44c905d6476e83be80eda3b1f65f as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/B/a68d44c905d6476e83be80eda3b1f65f 2024-11-18T20:12:45,611 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/B/a68d44c905d6476e83be80eda3b1f65f, entries=150, sequenceid=262, filesize=12.0 K 2024-11-18T20:12:45,612 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/.tmp/C/d2bbddc6235345a69a5b3754bdae8661 as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/C/d2bbddc6235345a69a5b3754bdae8661 2024-11-18T20:12:45,619 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/C/d2bbddc6235345a69a5b3754bdae8661, entries=150, sequenceid=262, filesize=12.0 K 2024-11-18T20:12:45,620 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=167.72 KB/171750 for 9d5f6ceb1ccdf5b797c44f37788d88e8 in 1690ms, sequenceid=262, compaction requested=false 2024-11-18T20:12:45,620 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2603): Flush status journal for 9d5f6ceb1ccdf5b797c44f37788d88e8: 2024-11-18T20:12:45,620 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8. 2024-11-18T20:12:45,620 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=49 2024-11-18T20:12:45,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.HMaster(4169): Remote procedure done, pid=49 2024-11-18T20:12:45,623 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=49, resume processing ppid=48 2024-11-18T20:12:45,623 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=49, ppid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.4550 sec 2024-11-18T20:12:45,628 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees in 2.4600 sec 2024-11-18T20:12:45,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(8855): Flush requested on 9d5f6ceb1ccdf5b797c44f37788d88e8 2024-11-18T20:12:45,905 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-18T20:12:45,905 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9d5f6ceb1ccdf5b797c44f37788d88e8 3/3 column families, dataSize=174.43 KB heapSize=457.78 KB 2024-11-18T20:12:45,924 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:45,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47158 deadline: 1731960825920, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:45,927 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:45,927 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:45,927 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:12:45,932 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118a149197026d64f20a94f1e136e5b39c6_9d5f6ceb1ccdf5b797c44f37788d88e8 is 50, key is test_row_0/A:col10/1731960764760/Put/seqid=0 2024-11-18T20:12:45,954 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742035_1211 (size=14994) 2024-11-18T20:12:45,957 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:45,965 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118a149197026d64f20a94f1e136e5b39c6_9d5f6ceb1ccdf5b797c44f37788d88e8 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118a149197026d64f20a94f1e136e5b39c6_9d5f6ceb1ccdf5b797c44f37788d88e8 2024-11-18T20:12:45,969 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/.tmp/A/1e440ee4e374495aae3c83f65d0bf36e, store: [table=TestAcidGuarantees family=A region=9d5f6ceb1ccdf5b797c44f37788d88e8] 2024-11-18T20:12:45,970 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/.tmp/A/1e440ee4e374495aae3c83f65d0bf36e is 175, key is test_row_0/A:col10/1731960764760/Put/seqid=0 2024-11-18T20:12:45,982 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742036_1212 (size=39949) 2024-11-18T20:12:45,983 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=291, memsize=58.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/.tmp/A/1e440ee4e374495aae3c83f65d0bf36e 2024-11-18T20:12:46,003 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/.tmp/B/111429484d764ec88cbb39f48033246d is 50, key is test_row_0/B:col10/1731960764760/Put/seqid=0 2024-11-18T20:12:46,015 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742037_1213 (size=12301) 2024-11-18T20:12:46,018 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58.14 KB at sequenceid=291 (bloomFilter=true), to=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/.tmp/B/111429484d764ec88cbb39f48033246d 2024-11-18T20:12:46,035 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/.tmp/C/69af93cc11e44ad2949b24d01fc87e65 is 50, key is test_row_0/C:col10/1731960764760/Put/seqid=0 2024-11-18T20:12:46,040 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:46,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47158 deadline: 1731960826033, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:46,041 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:46,041 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:46,041 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:12:46,067 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742038_1214 (size=12301) 2024-11-18T20:12:46,068 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58.14 KB at sequenceid=291 (bloomFilter=true), to=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/.tmp/C/69af93cc11e44ad2949b24d01fc87e65 2024-11-18T20:12:46,079 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/.tmp/A/1e440ee4e374495aae3c83f65d0bf36e as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/A/1e440ee4e374495aae3c83f65d0bf36e 2024-11-18T20:12:46,086 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/A/1e440ee4e374495aae3c83f65d0bf36e, entries=200, sequenceid=291, filesize=39.0 K 2024-11-18T20:12:46,088 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/.tmp/B/111429484d764ec88cbb39f48033246d as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/B/111429484d764ec88cbb39f48033246d 2024-11-18T20:12:46,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:46,100 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/B/111429484d764ec88cbb39f48033246d, entries=150, sequenceid=291, filesize=12.0 K 2024-11-18T20:12:46,102 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/.tmp/C/69af93cc11e44ad2949b24d01fc87e65 as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/C/69af93cc11e44ad2949b24d01fc87e65 2024-11-18T20:12:46,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:46,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:46,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:46,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:46,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:46,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:46,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:46,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:46,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:46,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:46,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:46,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:46,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:46,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:46,115 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/C/69af93cc11e44ad2949b24d01fc87e65, entries=150, sequenceid=291, filesize=12.0 K 2024-11-18T20:12:46,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:46,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:46,117 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~174.43 KB/178620, heapSize ~457.73 KB/468720, currentSize=26.84 KB/27480 for 9d5f6ceb1ccdf5b797c44f37788d88e8 in 212ms, sequenceid=291, compaction requested=true 2024-11-18T20:12:46,117 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9d5f6ceb1ccdf5b797c44f37788d88e8: 2024-11-18T20:12:46,117 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9d5f6ceb1ccdf5b797c44f37788d88e8:A, priority=-2147483648, current under compaction store size is 1 2024-11-18T20:12:46,117 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T20:12:46,117 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:12:46,117 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9d5f6ceb1ccdf5b797c44f37788d88e8:B, priority=-2147483648, current under compaction store size is 2 2024-11-18T20:12:46,118 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:12:46,118 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T20:12:46,118 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9d5f6ceb1ccdf5b797c44f37788d88e8:C, priority=-2147483648, current under compaction store size is 3 2024-11-18T20:12:46,118 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T20:12:46,119 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37283 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T20:12:46,119 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HStore(1541): 9d5f6ceb1ccdf5b797c44f37788d88e8/B is initiating minor compaction (all files) 2024-11-18T20:12:46,119 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9d5f6ceb1ccdf5b797c44f37788d88e8/B in TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8. 2024-11-18T20:12:46,119 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/B/800fa60bdfe242acbe36cb89ebbc3e5a, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/B/a68d44c905d6476e83be80eda3b1f65f, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/B/111429484d764ec88cbb39f48033246d] into tmpdir=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/.tmp, totalSize=36.4 K 2024-11-18T20:12:46,120 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102839 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T20:12:46,120 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HStore(1541): 9d5f6ceb1ccdf5b797c44f37788d88e8/A is initiating minor compaction (all files) 2024-11-18T20:12:46,120 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9d5f6ceb1ccdf5b797c44f37788d88e8/A in TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8. 2024-11-18T20:12:46,120 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/A/d17bc84cdc69479dbea8e28c1714df64, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/A/079e3ea2fd8b40739e0a8f155a1c7204, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/A/1e440ee4e374495aae3c83f65d0bf36e] into tmpdir=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/.tmp, totalSize=100.4 K 2024-11-18T20:12:46,120 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8. 2024-11-18T20:12:46,120 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8. files: [hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/A/d17bc84cdc69479dbea8e28c1714df64, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/A/079e3ea2fd8b40739e0a8f155a1c7204, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/A/1e440ee4e374495aae3c83f65d0bf36e] 2024-11-18T20:12:46,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:46,120 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.Compactor(225): Compacting 800fa60bdfe242acbe36cb89ebbc3e5a, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=251, earliestPutTs=1731960762158 2024-11-18T20:12:46,121 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] compactions.Compactor(225): Compacting d17bc84cdc69479dbea8e28c1714df64, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=251, earliestPutTs=1731960762158 2024-11-18T20:12:46,121 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.Compactor(225): Compacting a68d44c905d6476e83be80eda3b1f65f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=262, earliestPutTs=1731960762517 2024-11-18T20:12:46,121 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.Compactor(225): Compacting 111429484d764ec88cbb39f48033246d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=291, earliestPutTs=1731960764759 2024-11-18T20:12:46,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:46,121 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] compactions.Compactor(225): Compacting 079e3ea2fd8b40739e0a8f155a1c7204, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=262, earliestPutTs=1731960762517 2024-11-18T20:12:46,122 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1e440ee4e374495aae3c83f65d0bf36e, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=291, earliestPutTs=1731960764757 2024-11-18T20:12:46,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:46,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:46,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:46,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:46,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:46,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:46,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:46,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:46,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:46,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:46,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:46,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:46,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:46,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:46,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:46,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:46,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:46,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:46,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:46,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:46,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:46,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:46,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:46,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:46,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:46,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:46,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:46,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:46,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:46,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:46,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:46,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:46,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:46,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:46,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:46,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:46,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:46,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:46,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:46,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:46,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:46,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:46,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:46,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:46,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:46,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:46,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:46,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:46,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:46,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:46,147 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9d5f6ceb1ccdf5b797c44f37788d88e8#B#compaction#179 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:12:46,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:46,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:46,148 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/.tmp/B/b458e88987894af8b007172128569f9d is 50, key is test_row_0/B:col10/1731960764760/Put/seqid=0 2024-11-18T20:12:46,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:46,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:46,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:46,150 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=9d5f6ceb1ccdf5b797c44f37788d88e8] 2024-11-18T20:12:46,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:46,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:46,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:46,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:46,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:46,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:46,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:46,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:46,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:46,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:46,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:46,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:46,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:46,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:46,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:46,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:46,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:46,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:46,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:46,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:46,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:46,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:46,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:46,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:46,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:46,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:46,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:46,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:46,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:46,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:46,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:46,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:46,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:46,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:46,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:46,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:46,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:46,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:46,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:46,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:46,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:46,182 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024111853c8bb635bef420190b193548cf26b9b_9d5f6ceb1ccdf5b797c44f37788d88e8 store=[table=TestAcidGuarantees family=A region=9d5f6ceb1ccdf5b797c44f37788d88e8] 2024-11-18T20:12:46,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:46,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:46,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:46,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:46,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:46,184 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024111853c8bb635bef420190b193548cf26b9b_9d5f6ceb1ccdf5b797c44f37788d88e8, store=[table=TestAcidGuarantees family=A region=9d5f6ceb1ccdf5b797c44f37788d88e8] 2024-11-18T20:12:46,184 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111853c8bb635bef420190b193548cf26b9b_9d5f6ceb1ccdf5b797c44f37788d88e8 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=9d5f6ceb1ccdf5b797c44f37788d88e8] 2024-11-18T20:12:46,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:46,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:46,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:46,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:46,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:46,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:46,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:46,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:46,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:46,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:46,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:46,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:46,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:46,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:46,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:46,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:46,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:46,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:46,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:46,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:46,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:46,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:46,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:46,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:46,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:46,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:46,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:46,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:46,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:46,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:46,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:46,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:46,206 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742039_1215 (size=12983) 2024-11-18T20:12:46,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:46,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:46,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:46,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:46,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:46,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:46,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:46,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:46,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:46,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:46,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:46,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:46,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:46,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:46,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:46,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:46,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:46,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:46,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:46,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:46,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:46,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:46,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:46,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:46,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:46,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:46,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:46,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:46,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:46,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:46,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:46,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:46,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:46,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:46,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:46,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:46,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:46,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:46,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:46,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:46,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:46,226 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742040_1216 (size=4469) 2024-11-18T20:12:46,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:46,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:46,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:46,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:46,234 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9d5f6ceb1ccdf5b797c44f37788d88e8#A#compaction#180 average throughput is 0.29 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:12:46,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:46,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:46,235 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/.tmp/A/0beb8117576a40f498a38bd31bebbde1 is 175, key is test_row_0/A:col10/1731960764760/Put/seqid=0 2024-11-18T20:12:46,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:46,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:46,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:46,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:46,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:46,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:46,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:46,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:46,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:46,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:46,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:46,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:46,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:46,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:46,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:46,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:46,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:46,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:46,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:46,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:46,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:46,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:46,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:46,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:46,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:46,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:46,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:46,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:46,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:46,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:46,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:46,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:46,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:46,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:46,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:46,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:46,270 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742041_1217 (size=31937) 2024-11-18T20:12:46,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:46,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:46,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:46,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:46,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:46,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:46,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:46,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:46,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:46,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:46,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:46,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:46,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:46,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:46,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:46,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:46,280 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/.tmp/A/0beb8117576a40f498a38bd31bebbde1 as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/A/0beb8117576a40f498a38bd31bebbde1 2024-11-18T20:12:46,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:46,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:46,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(8855): Flush requested on 9d5f6ceb1ccdf5b797c44f37788d88e8 2024-11-18T20:12:46,282 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-18T20:12:46,283 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9d5f6ceb1ccdf5b797c44f37788d88e8 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-18T20:12:46,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:46,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:46,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:46,288 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9d5f6ceb1ccdf5b797c44f37788d88e8/A of 9d5f6ceb1ccdf5b797c44f37788d88e8 into 0beb8117576a40f498a38bd31bebbde1(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:12:46,288 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9d5f6ceb1ccdf5b797c44f37788d88e8: 2024-11-18T20:12:46,288 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., storeName=9d5f6ceb1ccdf5b797c44f37788d88e8/A, priority=13, startTime=1731960766117; duration=0sec 2024-11-18T20:12:46,288 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T20:12:46,288 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9d5f6ceb1ccdf5b797c44f37788d88e8:A 2024-11-18T20:12:46,289 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T20:12:46,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:46,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:46,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:46,290 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37283 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T20:12:46,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:46,290 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HStore(1541): 9d5f6ceb1ccdf5b797c44f37788d88e8/C is initiating minor compaction (all files) 2024-11-18T20:12:46,290 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9d5f6ceb1ccdf5b797c44f37788d88e8/C in TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8. 2024-11-18T20:12:46,291 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/C/7d0afbc55b954b81a3dd35686040e709, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/C/d2bbddc6235345a69a5b3754bdae8661, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/C/69af93cc11e44ad2949b24d01fc87e65] into tmpdir=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/.tmp, totalSize=36.4 K 2024-11-18T20:12:46,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:46,291 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7d0afbc55b954b81a3dd35686040e709, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=251, earliestPutTs=1731960762158 2024-11-18T20:12:46,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:46,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:46,292 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] compactions.Compactor(225): Compacting d2bbddc6235345a69a5b3754bdae8661, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=262, earliestPutTs=1731960762517 2024-11-18T20:12:46,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:46,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:46,293 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] compactions.Compactor(225): Compacting 69af93cc11e44ad2949b24d01fc87e65, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=291, earliestPutTs=1731960764759 2024-11-18T20:12:46,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:46,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:46,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:46,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:46,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:46,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:46,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:46,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:46,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:46,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:46,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:46,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:46,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:46,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:46,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:46,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:46,302 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118480747fc42754e479a9faf4574c6013a_9d5f6ceb1ccdf5b797c44f37788d88e8 is 50, key is test_row_0/A:col10/1731960766281/Put/seqid=0 2024-11-18T20:12:46,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:46,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:46,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:46,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:46,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:46,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:46,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:46,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:46,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:46,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:46,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:46,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:46,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:46,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:46,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:46,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:46,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:46,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:46,316 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9d5f6ceb1ccdf5b797c44f37788d88e8#C#compaction#182 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:12:46,316 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/.tmp/C/6037b029a7dd45e18b4221f6b25b3d42 is 50, key is test_row_0/C:col10/1731960764760/Put/seqid=0 2024-11-18T20:12:46,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:46,363 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742042_1218 (size=20074) 2024-11-18T20:12:46,385 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742043_1219 (size=12983) 2024-11-18T20:12:46,394 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/.tmp/C/6037b029a7dd45e18b4221f6b25b3d42 as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/C/6037b029a7dd45e18b4221f6b25b3d42 2024-11-18T20:12:46,405 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9d5f6ceb1ccdf5b797c44f37788d88e8/C of 9d5f6ceb1ccdf5b797c44f37788d88e8 into 6037b029a7dd45e18b4221f6b25b3d42(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:12:46,405 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9d5f6ceb1ccdf5b797c44f37788d88e8: 2024-11-18T20:12:46,405 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., storeName=9d5f6ceb1ccdf5b797c44f37788d88e8/C, priority=13, startTime=1731960766118; duration=0sec 2024-11-18T20:12:46,405 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:12:46,405 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9d5f6ceb1ccdf5b797c44f37788d88e8:C 2024-11-18T20:12:46,460 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:46,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 170 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47158 deadline: 1731960826460, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:46,461 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:46,461 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:46,461 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:12:46,566 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:46,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 171 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47158 deadline: 1731960826564, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:46,567 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:46,567 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:46,567 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:12:46,614 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/.tmp/B/b458e88987894af8b007172128569f9d as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/B/b458e88987894af8b007172128569f9d 2024-11-18T20:12:46,624 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9d5f6ceb1ccdf5b797c44f37788d88e8/B of 9d5f6ceb1ccdf5b797c44f37788d88e8 into b458e88987894af8b007172128569f9d(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:12:46,625 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9d5f6ceb1ccdf5b797c44f37788d88e8: 2024-11-18T20:12:46,625 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., storeName=9d5f6ceb1ccdf5b797c44f37788d88e8/B, priority=13, startTime=1731960766117; duration=0sec 2024-11-18T20:12:46,625 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:12:46,625 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9d5f6ceb1ccdf5b797c44f37788d88e8:B 2024-11-18T20:12:46,763 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:46,767 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118480747fc42754e479a9faf4574c6013a_9d5f6ceb1ccdf5b797c44f37788d88e8 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118480747fc42754e479a9faf4574c6013a_9d5f6ceb1ccdf5b797c44f37788d88e8 2024-11-18T20:12:46,770 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/.tmp/A/132c5b7993f348cf9abe82954140f5b1, store: [table=TestAcidGuarantees family=A region=9d5f6ceb1ccdf5b797c44f37788d88e8] 2024-11-18T20:12:46,770 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/.tmp/A/132c5b7993f348cf9abe82954140f5b1 is 175, key is test_row_0/A:col10/1731960766281/Put/seqid=0 2024-11-18T20:12:46,774 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742044_1220 (size=57333) 2024-11-18T20:12:46,780 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:46,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 172 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47158 deadline: 1731960826773, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:46,781 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:46,781 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:46,781 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:12:47,085 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:47,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 173 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47158 deadline: 1731960827083, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:47,085 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:47,085 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:47,085 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:12:47,175 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=302, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/.tmp/A/132c5b7993f348cf9abe82954140f5b1 2024-11-18T20:12:47,183 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/.tmp/B/dcf9fbf0aa3d492bb3146b6642e4f833 is 50, key is test_row_0/B:col10/1731960766281/Put/seqid=0 2024-11-18T20:12:47,192 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742045_1221 (size=12301) 2024-11-18T20:12:47,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-18T20:12:47,323 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-18T20:12:47,324 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-18T20:12:47,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] procedure2.ProcedureExecutor(1139): Stored pid=50, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees 2024-11-18T20:12:47,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-18T20:12:47,325 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=50, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-18T20:12:47,326 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=50, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-18T20:12:47,326 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=51, ppid=50, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-18T20:12:47,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-18T20:12:47,478 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35963 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-18T20:12:47,478 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8. 2024-11-18T20:12:47,478 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8. as already flushing 2024-11-18T20:12:47,478 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8. 2024-11-18T20:12:47,478 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 9d5f6ceb1ccdf5b797c44f37788d88e8, NAME => 'TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:12:47,478 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 9d5f6ceb1ccdf5b797c44f37788d88e8, NAME => 'TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:12:47,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9d5f6ceb1ccdf5b797c44f37788d88e8, NAME => 'TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9d5f6ceb1ccdf5b797c44f37788d88e8, NAME => 'TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:12:47,594 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=302 (bloomFilter=true), to=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/.tmp/B/dcf9fbf0aa3d492bb3146b6642e4f833 2024-11-18T20:12:47,595 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:47,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 174 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47158 deadline: 1731960827594, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:47,596 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:47,596 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:47,596 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:12:47,605 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/.tmp/C/40a05af8f8b54e7c8697c423b5fd490b is 50, key is test_row_0/C:col10/1731960766281/Put/seqid=0 2024-11-18T20:12:47,609 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742046_1222 (size=12301) 2024-11-18T20:12:47,631 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35963 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-18T20:12:47,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8. 2024-11-18T20:12:47,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8. as already flushing 2024-11-18T20:12:47,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8. 2024-11-18T20:12:47,631 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 9d5f6ceb1ccdf5b797c44f37788d88e8, NAME => 'TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:12:47,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 9d5f6ceb1ccdf5b797c44f37788d88e8, NAME => 'TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:12:47,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9d5f6ceb1ccdf5b797c44f37788d88e8, NAME => 'TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9d5f6ceb1ccdf5b797c44f37788d88e8, NAME => 'TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:12:47,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-18T20:12:47,743 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:47,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47184 deadline: 1731960827743, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:47,744 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to c0a89b2656d4,35963,1731960718223 for 'test_row_0' in 9d5f6ceb1ccdf5b797c44f37788d88e8 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8200 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-18T20:12:47,744 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:47,745 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:47,745 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:12:47,755 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:47,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47220 deadline: 1731960827753, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:47,756 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to c0a89b2656d4,35963,1731960718223 for 'test_row_1' in 9d5f6ceb1ccdf5b797c44f37788d88e8 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8213 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-18T20:12:47,756 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:47,756 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:47,756 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:12:47,784 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35963 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-18T20:12:47,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8. 2024-11-18T20:12:47,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8. as already flushing 2024-11-18T20:12:47,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8. 2024-11-18T20:12:47,784 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 9d5f6ceb1ccdf5b797c44f37788d88e8, NAME => 'TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:12:47,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 9d5f6ceb1ccdf5b797c44f37788d88e8, NAME => 'TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:12:47,784 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:47,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47206 deadline: 1731960827783, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:47,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9d5f6ceb1ccdf5b797c44f37788d88e8, NAME => 'TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9d5f6ceb1ccdf5b797c44f37788d88e8, NAME => 'TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:12:47,785 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to c0a89b2656d4,35963,1731960718223 for 'test_row_0' in 9d5f6ceb1ccdf5b797c44f37788d88e8 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8241 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-18T20:12:47,785 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:47,785 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:47,785 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:12:47,936 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35963 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-18T20:12:47,937 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8. 2024-11-18T20:12:47,937 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8. as already flushing 2024-11-18T20:12:47,937 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8. 2024-11-18T20:12:47,937 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 9d5f6ceb1ccdf5b797c44f37788d88e8, NAME => 'TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:12:47,937 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 9d5f6ceb1ccdf5b797c44f37788d88e8, NAME => 'TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:12:47,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9d5f6ceb1ccdf5b797c44f37788d88e8, NAME => 'TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9d5f6ceb1ccdf5b797c44f37788d88e8, NAME => 'TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:12:47,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-18T20:12:48,010 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=302 (bloomFilter=true), to=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/.tmp/C/40a05af8f8b54e7c8697c423b5fd490b 2024-11-18T20:12:48,014 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/.tmp/A/132c5b7993f348cf9abe82954140f5b1 as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/A/132c5b7993f348cf9abe82954140f5b1 2024-11-18T20:12:48,018 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/A/132c5b7993f348cf9abe82954140f5b1, entries=300, sequenceid=302, filesize=56.0 K 2024-11-18T20:12:48,019 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/.tmp/B/dcf9fbf0aa3d492bb3146b6642e4f833 as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/B/dcf9fbf0aa3d492bb3146b6642e4f833 2024-11-18T20:12:48,023 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/B/dcf9fbf0aa3d492bb3146b6642e4f833, entries=150, sequenceid=302, filesize=12.0 K 2024-11-18T20:12:48,024 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/.tmp/C/40a05af8f8b54e7c8697c423b5fd490b as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/C/40a05af8f8b54e7c8697c423b5fd490b 2024-11-18T20:12:48,029 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/C/40a05af8f8b54e7c8697c423b5fd490b, entries=150, sequenceid=302, filesize=12.0 K 2024-11-18T20:12:48,030 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 9d5f6ceb1ccdf5b797c44f37788d88e8 in 1748ms, sequenceid=302, compaction requested=false 2024-11-18T20:12:48,030 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9d5f6ceb1ccdf5b797c44f37788d88e8: 2024-11-18T20:12:48,089 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35963 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-18T20:12:48,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8. 2024-11-18T20:12:48,090 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2902): Flushing 9d5f6ceb1ccdf5b797c44f37788d88e8 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-18T20:12:48,097 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411185c4f3aa4378a45f2bd643fec3e2e0412_9d5f6ceb1ccdf5b797c44f37788d88e8 is 50, key is test_row_0/A:col10/1731960766440/Put/seqid=0 2024-11-18T20:12:48,104 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742047_1223 (size=12454) 2024-11-18T20:12:48,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-18T20:12:48,503 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:48,508 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411185c4f3aa4378a45f2bd643fec3e2e0412_9d5f6ceb1ccdf5b797c44f37788d88e8 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411185c4f3aa4378a45f2bd643fec3e2e0412_9d5f6ceb1ccdf5b797c44f37788d88e8 2024-11-18T20:12:48,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/.tmp/A/7cabbd366d234344ac9d941b78c3a831, store: [table=TestAcidGuarantees family=A region=9d5f6ceb1ccdf5b797c44f37788d88e8] 2024-11-18T20:12:48,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/.tmp/A/7cabbd366d234344ac9d941b78c3a831 is 175, key is test_row_0/A:col10/1731960766440/Put/seqid=0 2024-11-18T20:12:48,515 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742048_1224 (size=31255) 2024-11-18T20:12:48,605 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8. as already flushing 2024-11-18T20:12:48,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(8855): Flush requested on 9d5f6ceb1ccdf5b797c44f37788d88e8 2024-11-18T20:12:48,640 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:48,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 183 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47158 deadline: 1731960828640, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:48,641 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:48,641 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:48,641 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:12:48,746 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:48,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 184 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47158 deadline: 1731960828743, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:48,747 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:48,747 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:48,747 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:12:48,906 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:48,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47182 deadline: 1731960828903, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:48,906 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to c0a89b2656d4,35963,1731960718223 for 'test_row_2' in 9d5f6ceb1ccdf5b797c44f37788d88e8 of TestAcidGuarantees failed, tries = 9, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 18283 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-18T20:12:48,907 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:48,907 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:48,907 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:12:48,916 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=330, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/.tmp/A/7cabbd366d234344ac9d941b78c3a831 2024-11-18T20:12:48,923 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/.tmp/B/944b2989e5dd45d1abf25c5f89f1658c is 50, key is test_row_0/B:col10/1731960766440/Put/seqid=0 2024-11-18T20:12:48,937 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742049_1225 (size=12301) 2024-11-18T20:12:48,957 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:48,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 185 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47158 deadline: 1731960828953, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:48,957 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:48,958 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:48,958 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:12:49,265 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:49,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 186 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47158 deadline: 1731960829263, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:49,270 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:49,270 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:49,270 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:12:49,338 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=330 (bloomFilter=true), to=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/.tmp/B/944b2989e5dd45d1abf25c5f89f1658c 2024-11-18T20:12:49,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/.tmp/C/0c89084eb8384f8aad2c1f61d7040188 is 50, key is test_row_0/C:col10/1731960766440/Put/seqid=0 2024-11-18T20:12:49,353 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742050_1226 (size=12301) 2024-11-18T20:12:49,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-18T20:12:49,754 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=330 (bloomFilter=true), to=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/.tmp/C/0c89084eb8384f8aad2c1f61d7040188 2024-11-18T20:12:49,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/.tmp/A/7cabbd366d234344ac9d941b78c3a831 as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/A/7cabbd366d234344ac9d941b78c3a831 2024-11-18T20:12:49,769 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/A/7cabbd366d234344ac9d941b78c3a831, entries=150, sequenceid=330, filesize=30.5 K 2024-11-18T20:12:49,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/.tmp/B/944b2989e5dd45d1abf25c5f89f1658c as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/B/944b2989e5dd45d1abf25c5f89f1658c 2024-11-18T20:12:49,778 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/B/944b2989e5dd45d1abf25c5f89f1658c, entries=150, sequenceid=330, filesize=12.0 K 2024-11-18T20:12:49,779 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:12:49,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 187 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47158 deadline: 1731960829773, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:49,780 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:49,780 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:12:49,780 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:12:49,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/.tmp/C/0c89084eb8384f8aad2c1f61d7040188 as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/C/0c89084eb8384f8aad2c1f61d7040188 2024-11-18T20:12:49,788 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/C/0c89084eb8384f8aad2c1f61d7040188, entries=150, sequenceid=330, filesize=12.0 K 2024-11-18T20:12:49,789 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 9d5f6ceb1ccdf5b797c44f37788d88e8 in 1699ms, sequenceid=330, compaction requested=true 2024-11-18T20:12:49,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2603): Flush status journal for 9d5f6ceb1ccdf5b797c44f37788d88e8: 2024-11-18T20:12:49,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8. 2024-11-18T20:12:49,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=51 2024-11-18T20:12:49,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.HMaster(4169): Remote procedure done, pid=51 2024-11-18T20:12:49,797 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=51, resume processing ppid=50 2024-11-18T20:12:49,797 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=51, ppid=50, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.4650 sec 2024-11-18T20:12:49,799 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=50, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees in 2.4740 sec 2024-11-18T20:12:50,519 INFO [Thread-678 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-678. 2024-11-18T20:12:50,519 INFO [Thread-682 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-682. 2024-11-18T20:12:50,519 DEBUG [Thread-678 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-18T20:12:50,519 DEBUG [Thread-682 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-18T20:12:50,519 DEBUG [Thread-678 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:12:50,519 DEBUG [Thread-682 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:12:50,519 DEBUG [Thread-678 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:12:50,519 DEBUG [Thread-682 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:12:50,519 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T20:12:50,519 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T20:12:50,520 INFO [Thread-680 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-680. 2024-11-18T20:12:50,520 DEBUG [Thread-680 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-18T20:12:50,520 DEBUG [Thread-680 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:12:50,520 DEBUG [Thread-680 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:12:50,520 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T20:12:50,524 INFO [Thread-684 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-684. 2024-11-18T20:12:50,524 DEBUG [Thread-684 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-18T20:12:50,524 DEBUG [Thread-684 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:12:50,524 DEBUG [Thread-684 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:12:50,524 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T20:12:50,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(8855): Flush requested on 9d5f6ceb1ccdf5b797c44f37788d88e8 2024-11-18T20:12:50,795 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-18T20:12:50,795 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9d5f6ceb1ccdf5b797c44f37788d88e8 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-18T20:12:50,795 INFO [Thread-667 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-667. 2024-11-18T20:12:50,795 DEBUG [Thread-667 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-18T20:12:50,795 DEBUG [Thread-667 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:12:50,795 DEBUG [Thread-667 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:12:50,795 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T20:12:50,804 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411184e7fdd3d0c0f4592b9281e85c5d73302_9d5f6ceb1ccdf5b797c44f37788d88e8 is 50, key is test_row_0/A:col10/1731960768630/Put/seqid=0 2024-11-18T20:12:50,815 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742051_1227 (size=12454) 2024-11-18T20:12:50,816 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:50,821 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411184e7fdd3d0c0f4592b9281e85c5d73302_9d5f6ceb1ccdf5b797c44f37788d88e8 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411184e7fdd3d0c0f4592b9281e85c5d73302_9d5f6ceb1ccdf5b797c44f37788d88e8 2024-11-18T20:12:50,822 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/.tmp/A/48f53753861c4e59a9b4c8faaaf22191, store: [table=TestAcidGuarantees family=A region=9d5f6ceb1ccdf5b797c44f37788d88e8] 2024-11-18T20:12:50,823 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/.tmp/A/48f53753861c4e59a9b4c8faaaf22191 is 175, key is test_row_0/A:col10/1731960768630/Put/seqid=0 2024-11-18T20:12:50,833 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742052_1228 (size=31255) 2024-11-18T20:12:50,834 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=342, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/.tmp/A/48f53753861c4e59a9b4c8faaaf22191 2024-11-18T20:12:50,846 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/.tmp/B/bd6d8cccd2a340c8892eb9c7980aef18 is 50, key is test_row_0/B:col10/1731960768630/Put/seqid=0 2024-11-18T20:12:50,850 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742053_1229 (size=12301) 2024-11-18T20:12:51,250 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=342 (bloomFilter=true), to=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/.tmp/B/bd6d8cccd2a340c8892eb9c7980aef18 2024-11-18T20:12:51,257 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/.tmp/C/36e133405d354127b3a2ff033fe4bd2a is 50, key is test_row_0/C:col10/1731960768630/Put/seqid=0 2024-11-18T20:12:51,263 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742054_1230 (size=12301) 2024-11-18T20:12:51,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-18T20:12:51,493 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-18T20:12:51,664 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=342 (bloomFilter=true), to=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/.tmp/C/36e133405d354127b3a2ff033fe4bd2a 2024-11-18T20:12:51,669 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/.tmp/A/48f53753861c4e59a9b4c8faaaf22191 as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/A/48f53753861c4e59a9b4c8faaaf22191 2024-11-18T20:12:51,673 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/A/48f53753861c4e59a9b4c8faaaf22191, entries=150, sequenceid=342, filesize=30.5 K 2024-11-18T20:12:51,674 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/.tmp/B/bd6d8cccd2a340c8892eb9c7980aef18 as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/B/bd6d8cccd2a340c8892eb9c7980aef18 2024-11-18T20:12:51,679 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/B/bd6d8cccd2a340c8892eb9c7980aef18, entries=150, sequenceid=342, filesize=12.0 K 2024-11-18T20:12:51,680 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/.tmp/C/36e133405d354127b3a2ff033fe4bd2a as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/C/36e133405d354127b3a2ff033fe4bd2a 2024-11-18T20:12:51,685 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/C/36e133405d354127b3a2ff033fe4bd2a, entries=150, sequenceid=342, filesize=12.0 K 2024-11-18T20:12:51,686 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=0 B/0 for 9d5f6ceb1ccdf5b797c44f37788d88e8 in 891ms, sequenceid=342, compaction requested=true 2024-11-18T20:12:51,686 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9d5f6ceb1ccdf5b797c44f37788d88e8: 2024-11-18T20:12:51,687 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9d5f6ceb1ccdf5b797c44f37788d88e8:A, priority=-2147483648, current under compaction store size is 1 2024-11-18T20:12:51,687 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:12:51,687 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9d5f6ceb1ccdf5b797c44f37788d88e8:B, priority=-2147483648, current under compaction store size is 2 2024-11-18T20:12:51,687 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-18T20:12:51,687 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T20:12:51,687 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9d5f6ceb1ccdf5b797c44f37788d88e8:C, priority=-2147483648, current under compaction store size is 3 2024-11-18T20:12:51,687 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-18T20:12:51,687 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-18T20:12:51,688 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 151780 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-18T20:12:51,688 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49886 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-18T20:12:51,688 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HStore(1541): 9d5f6ceb1ccdf5b797c44f37788d88e8/A is initiating minor compaction (all files) 2024-11-18T20:12:51,688 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HStore(1541): 9d5f6ceb1ccdf5b797c44f37788d88e8/B is initiating minor compaction (all files) 2024-11-18T20:12:51,688 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9d5f6ceb1ccdf5b797c44f37788d88e8/A in TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8. 2024-11-18T20:12:51,688 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9d5f6ceb1ccdf5b797c44f37788d88e8/B in TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8. 2024-11-18T20:12:51,688 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/B/b458e88987894af8b007172128569f9d, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/B/dcf9fbf0aa3d492bb3146b6642e4f833, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/B/944b2989e5dd45d1abf25c5f89f1658c, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/B/bd6d8cccd2a340c8892eb9c7980aef18] into tmpdir=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/.tmp, totalSize=48.7 K 2024-11-18T20:12:51,688 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/A/0beb8117576a40f498a38bd31bebbde1, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/A/132c5b7993f348cf9abe82954140f5b1, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/A/7cabbd366d234344ac9d941b78c3a831, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/A/48f53753861c4e59a9b4c8faaaf22191] into tmpdir=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/.tmp, totalSize=148.2 K 2024-11-18T20:12:51,688 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8. 2024-11-18T20:12:51,688 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8. files: [hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/A/0beb8117576a40f498a38bd31bebbde1, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/A/132c5b7993f348cf9abe82954140f5b1, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/A/7cabbd366d234344ac9d941b78c3a831, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/A/48f53753861c4e59a9b4c8faaaf22191] 2024-11-18T20:12:51,689 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.Compactor(225): Compacting b458e88987894af8b007172128569f9d, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=291, earliestPutTs=1731960764759 2024-11-18T20:12:51,689 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0beb8117576a40f498a38bd31bebbde1, keycount=150, bloomtype=ROW, size=31.2 K, encoding=NONE, compression=NONE, seqNum=291, earliestPutTs=1731960764759 2024-11-18T20:12:51,689 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.Compactor(225): Compacting dcf9fbf0aa3d492bb3146b6642e4f833, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=302, earliestPutTs=1731960765910 2024-11-18T20:12:51,689 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] compactions.Compactor(225): Compacting 132c5b7993f348cf9abe82954140f5b1, keycount=300, bloomtype=ROW, size=56.0 K, encoding=NONE, compression=NONE, seqNum=302, earliestPutTs=1731960765910 2024-11-18T20:12:51,689 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.Compactor(225): Compacting 944b2989e5dd45d1abf25c5f89f1658c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=330, earliestPutTs=1731960766428 2024-11-18T20:12:51,689 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7cabbd366d234344ac9d941b78c3a831, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=330, earliestPutTs=1731960766428 2024-11-18T20:12:51,689 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.Compactor(225): Compacting bd6d8cccd2a340c8892eb9c7980aef18, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=342, earliestPutTs=1731960768630 2024-11-18T20:12:51,689 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] compactions.Compactor(225): Compacting 48f53753861c4e59a9b4c8faaaf22191, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=342, earliestPutTs=1731960768630 2024-11-18T20:12:51,700 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=9d5f6ceb1ccdf5b797c44f37788d88e8] 2024-11-18T20:12:51,702 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241118436c6be665c54ae3934c4a249a03af8c_9d5f6ceb1ccdf5b797c44f37788d88e8 store=[table=TestAcidGuarantees family=A region=9d5f6ceb1ccdf5b797c44f37788d88e8] 2024-11-18T20:12:51,703 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9d5f6ceb1ccdf5b797c44f37788d88e8#B#compaction#191 average throughput is 0.94 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:12:51,703 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/.tmp/B/0c51cf40051b4fa9b5ea10dded4e1f43 is 50, key is test_row_0/B:col10/1731960768630/Put/seqid=0 2024-11-18T20:12:51,719 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742055_1231 (size=13119) 2024-11-18T20:12:51,732 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241118436c6be665c54ae3934c4a249a03af8c_9d5f6ceb1ccdf5b797c44f37788d88e8, store=[table=TestAcidGuarantees family=A region=9d5f6ceb1ccdf5b797c44f37788d88e8] 2024-11-18T20:12:51,732 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118436c6be665c54ae3934c4a249a03af8c_9d5f6ceb1ccdf5b797c44f37788d88e8 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=9d5f6ceb1ccdf5b797c44f37788d88e8] 2024-11-18T20:12:51,736 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742056_1232 (size=4469) 2024-11-18T20:12:52,127 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/.tmp/B/0c51cf40051b4fa9b5ea10dded4e1f43 as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/B/0c51cf40051b4fa9b5ea10dded4e1f43 2024-11-18T20:12:52,134 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 9d5f6ceb1ccdf5b797c44f37788d88e8/B of 9d5f6ceb1ccdf5b797c44f37788d88e8 into 0c51cf40051b4fa9b5ea10dded4e1f43(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:12:52,134 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9d5f6ceb1ccdf5b797c44f37788d88e8: 2024-11-18T20:12:52,134 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., storeName=9d5f6ceb1ccdf5b797c44f37788d88e8/B, priority=12, startTime=1731960771687; duration=0sec 2024-11-18T20:12:52,134 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T20:12:52,134 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9d5f6ceb1ccdf5b797c44f37788d88e8:B 2024-11-18T20:12:52,134 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-18T20:12:52,136 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49886 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-18T20:12:52,136 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HStore(1541): 9d5f6ceb1ccdf5b797c44f37788d88e8/C is initiating minor compaction (all files) 2024-11-18T20:12:52,136 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9d5f6ceb1ccdf5b797c44f37788d88e8/C in TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8. 2024-11-18T20:12:52,136 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/C/6037b029a7dd45e18b4221f6b25b3d42, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/C/40a05af8f8b54e7c8697c423b5fd490b, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/C/0c89084eb8384f8aad2c1f61d7040188, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/C/36e133405d354127b3a2ff033fe4bd2a] into tmpdir=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/.tmp, totalSize=48.7 K 2024-11-18T20:12:52,137 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.Compactor(225): Compacting 6037b029a7dd45e18b4221f6b25b3d42, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=291, earliestPutTs=1731960764759 2024-11-18T20:12:52,138 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.Compactor(225): Compacting 40a05af8f8b54e7c8697c423b5fd490b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=302, earliestPutTs=1731960765910 2024-11-18T20:12:52,138 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9d5f6ceb1ccdf5b797c44f37788d88e8#A#compaction#192 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:12:52,138 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.Compactor(225): Compacting 0c89084eb8384f8aad2c1f61d7040188, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=330, earliestPutTs=1731960766428 2024-11-18T20:12:52,139 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.Compactor(225): Compacting 36e133405d354127b3a2ff033fe4bd2a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=342, earliestPutTs=1731960768630 2024-11-18T20:12:52,139 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/.tmp/A/7a4c9c2b5da54ed6a716bd200c941207 is 175, key is test_row_0/A:col10/1731960768630/Put/seqid=0 2024-11-18T20:12:52,144 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742057_1233 (size=32073) 2024-11-18T20:12:52,151 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9d5f6ceb1ccdf5b797c44f37788d88e8#C#compaction#193 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:12:52,151 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/.tmp/C/3419e63ff3af43cd9201eee2f8fedbd1 is 50, key is test_row_0/C:col10/1731960768630/Put/seqid=0 2024-11-18T20:12:52,155 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742058_1234 (size=13119) 2024-11-18T20:12:52,554 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/.tmp/A/7a4c9c2b5da54ed6a716bd200c941207 as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/A/7a4c9c2b5da54ed6a716bd200c941207 2024-11-18T20:12:52,562 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 9d5f6ceb1ccdf5b797c44f37788d88e8/A of 9d5f6ceb1ccdf5b797c44f37788d88e8 into 7a4c9c2b5da54ed6a716bd200c941207(size=31.3 K), total size for store is 31.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:12:52,562 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9d5f6ceb1ccdf5b797c44f37788d88e8: 2024-11-18T20:12:52,562 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., storeName=9d5f6ceb1ccdf5b797c44f37788d88e8/A, priority=12, startTime=1731960771686; duration=0sec 2024-11-18T20:12:52,562 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/.tmp/C/3419e63ff3af43cd9201eee2f8fedbd1 as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/C/3419e63ff3af43cd9201eee2f8fedbd1 2024-11-18T20:12:52,563 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:12:52,563 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9d5f6ceb1ccdf5b797c44f37788d88e8:A 2024-11-18T20:12:52,569 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 9d5f6ceb1ccdf5b797c44f37788d88e8/C of 9d5f6ceb1ccdf5b797c44f37788d88e8 into 3419e63ff3af43cd9201eee2f8fedbd1(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:12:52,569 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9d5f6ceb1ccdf5b797c44f37788d88e8: 2024-11-18T20:12:52,569 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., storeName=9d5f6ceb1ccdf5b797c44f37788d88e8/C, priority=12, startTime=1731960771687; duration=0sec 2024-11-18T20:12:52,569 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:12:52,569 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9d5f6ceb1ccdf5b797c44f37788d88e8:C 2024-11-18T20:12:56,533 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-18T20:12:57,785 INFO [Thread-671 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-671. 2024-11-18T20:12:57,785 DEBUG [Thread-671 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-18T20:12:57,786 DEBUG [Thread-671 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:12:57,786 DEBUG [Thread-671 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:12:57,786 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T20:12:57,804 INFO [Thread-675 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-675. 2024-11-18T20:12:57,804 DEBUG [Thread-675 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-18T20:12:57,804 DEBUG [Thread-675 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:12:57,804 DEBUG [Thread-675 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:12:57,804 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T20:12:57,814 INFO [Thread-669 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-669. 2024-11-18T20:12:57,814 DEBUG [Thread-669 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-18T20:12:57,814 DEBUG [Thread-669 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:12:57,814 DEBUG [Thread-669 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:12:57,814 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T20:12:58,977 INFO [Thread-673 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-673. 2024-11-18T20:12:58,977 DEBUG [Thread-673 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-18T20:12:58,977 DEBUG [Thread-673 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:12:58,977 DEBUG [Thread-673 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:12:58,978 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T20:12:58,978 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-18T20:12:58,978 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 135 2024-11-18T20:12:58,978 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 39 2024-11-18T20:12:58,978 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 46 2024-11-18T20:12:58,978 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 7 2024-11-18T20:12:58,978 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 43 2024-11-18T20:12:58,979 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-18T20:12:58,979 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3683 2024-11-18T20:12:58,979 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3711 2024-11-18T20:12:58,979 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-18T20:12:58,979 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1570 2024-11-18T20:12:58,979 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4707 rows 2024-11-18T20:12:58,979 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1579 2024-11-18T20:12:58,979 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4737 rows 2024-11-18T20:12:58,979 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-18T20:12:58,979 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobMixedAtomicity(AcidGuaranteesTestBase.java:132) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-18T20:12:58,979 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:12:58,980 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:12:58,980 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T20:12:58,982 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-18T20:12:58,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] procedure2.ProcedureExecutor(1139): Stored pid=52, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-18T20:12:58,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-18T20:12:58,986 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731960778986"}]},"ts":"1731960778986"} 2024-11-18T20:12:58,988 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-18T20:12:58,988 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-18T20:12:58,988 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=53, ppid=52, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-18T20:12:58,990 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=54, ppid=53, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=9d5f6ceb1ccdf5b797c44f37788d88e8, UNASSIGN}] 2024-11-18T20:12:58,990 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=54, ppid=53, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=9d5f6ceb1ccdf5b797c44f37788d88e8, UNASSIGN 2024-11-18T20:12:58,991 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=54 updating hbase:meta row=9d5f6ceb1ccdf5b797c44f37788d88e8, regionState=CLOSING, regionLocation=c0a89b2656d4,35963,1731960718223 2024-11-18T20:12:58,992 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=54, ppid=53, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=9d5f6ceb1ccdf5b797c44f37788d88e8, UNASSIGN because future has completed 2024-11-18T20:12:58,992 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-18T20:12:58,993 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=55, ppid=54, state=RUNNABLE, hasLock=false; CloseRegionProcedure 9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223}] 2024-11-18T20:12:59,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-18T20:12:59,145 INFO [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] handler.UnassignRegionHandler(122): Close 9d5f6ceb1ccdf5b797c44f37788d88e8 2024-11-18T20:12:59,145 DEBUG [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-18T20:12:59,145 DEBUG [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HRegion(1722): Closing 9d5f6ceb1ccdf5b797c44f37788d88e8, disabling compactions & flushes 2024-11-18T20:12:59,145 INFO [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8. 2024-11-18T20:12:59,145 DEBUG [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8. 2024-11-18T20:12:59,145 DEBUG [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8. after waiting 0 ms 2024-11-18T20:12:59,145 DEBUG [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8. 2024-11-18T20:12:59,145 INFO [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HRegion(2902): Flushing 9d5f6ceb1ccdf5b797c44f37788d88e8 3/3 column families, dataSize=26.84 KB heapSize=71.06 KB 2024-11-18T20:12:59,152 DEBUG [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111875e5410bf48b41199a33e2f1d2853f1e_9d5f6ceb1ccdf5b797c44f37788d88e8 is 50, key is test_row_0/A:col10/1731960777813/Put/seqid=0 2024-11-18T20:12:59,156 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742059_1235 (size=12454) 2024-11-18T20:12:59,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-18T20:12:59,557 DEBUG [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:12:59,567 INFO [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111875e5410bf48b41199a33e2f1d2853f1e_9d5f6ceb1ccdf5b797c44f37788d88e8 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111875e5410bf48b41199a33e2f1d2853f1e_9d5f6ceb1ccdf5b797c44f37788d88e8 2024-11-18T20:12:59,569 DEBUG [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/.tmp/A/9d1545bcc0054d9eae892e890f8c0ae8, store: [table=TestAcidGuarantees family=A region=9d5f6ceb1ccdf5b797c44f37788d88e8] 2024-11-18T20:12:59,570 DEBUG [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/.tmp/A/9d1545bcc0054d9eae892e890f8c0ae8 is 175, key is test_row_0/A:col10/1731960777813/Put/seqid=0 2024-11-18T20:12:59,573 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742060_1236 (size=31255) 2024-11-18T20:12:59,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-18T20:12:59,974 INFO [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=352, memsize=8.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/.tmp/A/9d1545bcc0054d9eae892e890f8c0ae8 2024-11-18T20:12:59,988 DEBUG [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/.tmp/B/44c0d64d5bf94a3da79a366c4492faba is 50, key is test_row_0/B:col10/1731960777813/Put/seqid=0 2024-11-18T20:12:59,992 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742061_1237 (size=12301) 2024-11-18T20:13:00,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-18T20:13:00,393 INFO [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=352 (bloomFilter=true), to=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/.tmp/B/44c0d64d5bf94a3da79a366c4492faba 2024-11-18T20:13:00,404 DEBUG [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/.tmp/C/d7d5f1184d8e4d23aa178efdc3682c6f is 50, key is test_row_0/C:col10/1731960777813/Put/seqid=0 2024-11-18T20:13:00,408 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742062_1238 (size=12301) 2024-11-18T20:13:00,809 INFO [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=352 (bloomFilter=true), to=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/.tmp/C/d7d5f1184d8e4d23aa178efdc3682c6f 2024-11-18T20:13:00,815 DEBUG [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/.tmp/A/9d1545bcc0054d9eae892e890f8c0ae8 as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/A/9d1545bcc0054d9eae892e890f8c0ae8 2024-11-18T20:13:00,820 INFO [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/A/9d1545bcc0054d9eae892e890f8c0ae8, entries=150, sequenceid=352, filesize=30.5 K 2024-11-18T20:13:00,821 DEBUG [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/.tmp/B/44c0d64d5bf94a3da79a366c4492faba as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/B/44c0d64d5bf94a3da79a366c4492faba 2024-11-18T20:13:00,826 INFO [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/B/44c0d64d5bf94a3da79a366c4492faba, entries=150, sequenceid=352, filesize=12.0 K 2024-11-18T20:13:00,827 DEBUG [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/.tmp/C/d7d5f1184d8e4d23aa178efdc3682c6f as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/C/d7d5f1184d8e4d23aa178efdc3682c6f 2024-11-18T20:13:00,832 INFO [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/C/d7d5f1184d8e4d23aa178efdc3682c6f, entries=150, sequenceid=352, filesize=12.0 K 2024-11-18T20:13:00,832 INFO [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~71.02 KB/72720, currentSize=0 B/0 for 9d5f6ceb1ccdf5b797c44f37788d88e8 in 1687ms, sequenceid=352, compaction requested=false 2024-11-18T20:13:00,833 DEBUG [StoreCloser-TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/A/c0f2285265cb49309b85d2fa7aec3ad1, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/A/c9f67eefaab1486680e5a996b6d8817b, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/A/156bcc007c634c6fa7315ad53c7623d7, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/A/2a02f4728fc24b718d041e65260cc24c, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/A/237de5ed6437485fb888eeb558983e15, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/A/ea6d38a7cd30486f98515c556d750e84, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/A/88bd77f166ab49d39e837bd064e2194d, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/A/82936db65f6b4993b25aa32eac0b21d7, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/A/ff255c12130b43e9a829043f35525ddf, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/A/894d0df15ec64e6fa3be363e61ce27db, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/A/5ae222a3e3734d61933f7895b0296150, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/A/787ada51f532477fb87d941fb3d151bd, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/A/95dba009994343528996c305712ce70d, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/A/ad0bd1618e0e4e7d80a473e735edbb71, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/A/7a393fd61b4a4e4a9a5f1450d4d17879, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/A/89e2d4925fd9481f952e2920f34c6983, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/A/d17bc84cdc69479dbea8e28c1714df64, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/A/e7b2ff26190f486c85a8871d7ebd7b66, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/A/079e3ea2fd8b40739e0a8f155a1c7204, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/A/1e440ee4e374495aae3c83f65d0bf36e, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/A/0beb8117576a40f498a38bd31bebbde1, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/A/132c5b7993f348cf9abe82954140f5b1, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/A/7cabbd366d234344ac9d941b78c3a831, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/A/48f53753861c4e59a9b4c8faaaf22191] to archive 2024-11-18T20:13:00,834 DEBUG [StoreCloser-TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-18T20:13:00,836 DEBUG [StoreCloser-TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/A/c0f2285265cb49309b85d2fa7aec3ad1 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/A/c0f2285265cb49309b85d2fa7aec3ad1 2024-11-18T20:13:00,838 DEBUG [StoreCloser-TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/A/c9f67eefaab1486680e5a996b6d8817b to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/A/c9f67eefaab1486680e5a996b6d8817b 2024-11-18T20:13:00,839 DEBUG [StoreCloser-TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/A/156bcc007c634c6fa7315ad53c7623d7 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/A/156bcc007c634c6fa7315ad53c7623d7 2024-11-18T20:13:00,840 DEBUG [StoreCloser-TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/A/2a02f4728fc24b718d041e65260cc24c to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/A/2a02f4728fc24b718d041e65260cc24c 2024-11-18T20:13:00,841 DEBUG [StoreCloser-TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/A/237de5ed6437485fb888eeb558983e15 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/A/237de5ed6437485fb888eeb558983e15 2024-11-18T20:13:00,842 DEBUG [StoreCloser-TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/A/ea6d38a7cd30486f98515c556d750e84 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/A/ea6d38a7cd30486f98515c556d750e84 2024-11-18T20:13:00,843 DEBUG [StoreCloser-TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/A/88bd77f166ab49d39e837bd064e2194d to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/A/88bd77f166ab49d39e837bd064e2194d 2024-11-18T20:13:00,844 DEBUG [StoreCloser-TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/A/82936db65f6b4993b25aa32eac0b21d7 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/A/82936db65f6b4993b25aa32eac0b21d7 2024-11-18T20:13:00,845 DEBUG [StoreCloser-TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/A/ff255c12130b43e9a829043f35525ddf to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/A/ff255c12130b43e9a829043f35525ddf 2024-11-18T20:13:00,846 DEBUG [StoreCloser-TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/A/894d0df15ec64e6fa3be363e61ce27db to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/A/894d0df15ec64e6fa3be363e61ce27db 2024-11-18T20:13:00,847 DEBUG [StoreCloser-TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/A/5ae222a3e3734d61933f7895b0296150 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/A/5ae222a3e3734d61933f7895b0296150 2024-11-18T20:13:00,848 DEBUG [StoreCloser-TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/A/787ada51f532477fb87d941fb3d151bd to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/A/787ada51f532477fb87d941fb3d151bd 2024-11-18T20:13:00,849 DEBUG [StoreCloser-TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/A/95dba009994343528996c305712ce70d to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/A/95dba009994343528996c305712ce70d 2024-11-18T20:13:00,850 DEBUG [StoreCloser-TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/A/ad0bd1618e0e4e7d80a473e735edbb71 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/A/ad0bd1618e0e4e7d80a473e735edbb71 2024-11-18T20:13:00,851 DEBUG [StoreCloser-TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/A/7a393fd61b4a4e4a9a5f1450d4d17879 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/A/7a393fd61b4a4e4a9a5f1450d4d17879 2024-11-18T20:13:00,852 DEBUG [StoreCloser-TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/A/89e2d4925fd9481f952e2920f34c6983 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/A/89e2d4925fd9481f952e2920f34c6983 2024-11-18T20:13:00,853 DEBUG [StoreCloser-TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/A/d17bc84cdc69479dbea8e28c1714df64 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/A/d17bc84cdc69479dbea8e28c1714df64 2024-11-18T20:13:00,854 DEBUG [StoreCloser-TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/A/e7b2ff26190f486c85a8871d7ebd7b66 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/A/e7b2ff26190f486c85a8871d7ebd7b66 2024-11-18T20:13:00,855 DEBUG [StoreCloser-TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/A/079e3ea2fd8b40739e0a8f155a1c7204 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/A/079e3ea2fd8b40739e0a8f155a1c7204 2024-11-18T20:13:00,856 DEBUG [StoreCloser-TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/A/1e440ee4e374495aae3c83f65d0bf36e to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/A/1e440ee4e374495aae3c83f65d0bf36e 2024-11-18T20:13:00,857 DEBUG [StoreCloser-TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/A/0beb8117576a40f498a38bd31bebbde1 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/A/0beb8117576a40f498a38bd31bebbde1 2024-11-18T20:13:00,858 DEBUG [StoreCloser-TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/A/132c5b7993f348cf9abe82954140f5b1 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/A/132c5b7993f348cf9abe82954140f5b1 2024-11-18T20:13:00,859 DEBUG [StoreCloser-TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/A/7cabbd366d234344ac9d941b78c3a831 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/A/7cabbd366d234344ac9d941b78c3a831 2024-11-18T20:13:00,860 DEBUG [StoreCloser-TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/A/48f53753861c4e59a9b4c8faaaf22191 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/A/48f53753861c4e59a9b4c8faaaf22191 2024-11-18T20:13:00,861 DEBUG [StoreCloser-TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/B/f1fbc10d3bee4ddc91795b98e07ec8a9, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/B/e8a7cc11d67b421f933954581e2cc452, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/B/dd5c5ee28e48453bb106b3662079efef, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/B/347b868f5fa84793b0ef2f608bfe6462, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/B/850fb40d28654286a3b8a66c3657b85d, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/B/105a322f94b241759e166e95d61e1a9b, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/B/591cbd36f0384cc4aba373b134f75759, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/B/676f1d9f947a49e98ed170a2cd6a1f95, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/B/c43632556cca49a8951a5f3661364dfe, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/B/8d2fa5da60c54e37802fccd9d8178247, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/B/1f6395b0486c41b2abd10e52df1b1d2c, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/B/56ce60d2bd7842ab84770ce977f26de4, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/B/ad154585b3934afcabd208d580fba23a, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/B/d528bbee6dbd422f859d63caa5f4f4cc, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/B/1b11a567514544a6b0a4cde5e2c37dcf, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/B/fab2f392ddfa4526a1505dc8b8fee5dd, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/B/800fa60bdfe242acbe36cb89ebbc3e5a, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/B/7a083120e0a64294921239d966d2e2bb, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/B/a68d44c905d6476e83be80eda3b1f65f, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/B/b458e88987894af8b007172128569f9d, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/B/111429484d764ec88cbb39f48033246d, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/B/dcf9fbf0aa3d492bb3146b6642e4f833, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/B/944b2989e5dd45d1abf25c5f89f1658c, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/B/bd6d8cccd2a340c8892eb9c7980aef18] to archive 2024-11-18T20:13:00,862 DEBUG [StoreCloser-TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-18T20:13:00,863 DEBUG [StoreCloser-TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/B/f1fbc10d3bee4ddc91795b98e07ec8a9 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/B/f1fbc10d3bee4ddc91795b98e07ec8a9 2024-11-18T20:13:00,864 DEBUG [StoreCloser-TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/B/e8a7cc11d67b421f933954581e2cc452 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/B/e8a7cc11d67b421f933954581e2cc452 2024-11-18T20:13:00,865 DEBUG [StoreCloser-TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/B/dd5c5ee28e48453bb106b3662079efef to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/B/dd5c5ee28e48453bb106b3662079efef 2024-11-18T20:13:00,866 DEBUG [StoreCloser-TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/B/347b868f5fa84793b0ef2f608bfe6462 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/B/347b868f5fa84793b0ef2f608bfe6462 2024-11-18T20:13:00,867 DEBUG [StoreCloser-TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/B/850fb40d28654286a3b8a66c3657b85d to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/B/850fb40d28654286a3b8a66c3657b85d 2024-11-18T20:13:00,868 DEBUG [StoreCloser-TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/B/105a322f94b241759e166e95d61e1a9b to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/B/105a322f94b241759e166e95d61e1a9b 2024-11-18T20:13:00,869 DEBUG [StoreCloser-TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/B/591cbd36f0384cc4aba373b134f75759 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/B/591cbd36f0384cc4aba373b134f75759 2024-11-18T20:13:00,870 DEBUG [StoreCloser-TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/B/676f1d9f947a49e98ed170a2cd6a1f95 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/B/676f1d9f947a49e98ed170a2cd6a1f95 2024-11-18T20:13:00,871 DEBUG [StoreCloser-TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/B/c43632556cca49a8951a5f3661364dfe to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/B/c43632556cca49a8951a5f3661364dfe 2024-11-18T20:13:00,872 DEBUG [StoreCloser-TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/B/8d2fa5da60c54e37802fccd9d8178247 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/B/8d2fa5da60c54e37802fccd9d8178247 2024-11-18T20:13:00,873 DEBUG [StoreCloser-TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/B/1f6395b0486c41b2abd10e52df1b1d2c to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/B/1f6395b0486c41b2abd10e52df1b1d2c 2024-11-18T20:13:00,874 DEBUG [StoreCloser-TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/B/56ce60d2bd7842ab84770ce977f26de4 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/B/56ce60d2bd7842ab84770ce977f26de4 2024-11-18T20:13:00,875 DEBUG [StoreCloser-TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/B/ad154585b3934afcabd208d580fba23a to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/B/ad154585b3934afcabd208d580fba23a 2024-11-18T20:13:00,876 DEBUG [StoreCloser-TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/B/d528bbee6dbd422f859d63caa5f4f4cc to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/B/d528bbee6dbd422f859d63caa5f4f4cc 2024-11-18T20:13:00,877 DEBUG [StoreCloser-TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/B/1b11a567514544a6b0a4cde5e2c37dcf to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/B/1b11a567514544a6b0a4cde5e2c37dcf 2024-11-18T20:13:00,878 DEBUG [StoreCloser-TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/B/fab2f392ddfa4526a1505dc8b8fee5dd to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/B/fab2f392ddfa4526a1505dc8b8fee5dd 2024-11-18T20:13:00,879 DEBUG [StoreCloser-TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/B/800fa60bdfe242acbe36cb89ebbc3e5a to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/B/800fa60bdfe242acbe36cb89ebbc3e5a 2024-11-18T20:13:00,880 DEBUG [StoreCloser-TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/B/7a083120e0a64294921239d966d2e2bb to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/B/7a083120e0a64294921239d966d2e2bb 2024-11-18T20:13:00,881 DEBUG [StoreCloser-TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/B/a68d44c905d6476e83be80eda3b1f65f to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/B/a68d44c905d6476e83be80eda3b1f65f 2024-11-18T20:13:00,882 DEBUG [StoreCloser-TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/B/b458e88987894af8b007172128569f9d to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/B/b458e88987894af8b007172128569f9d 2024-11-18T20:13:00,883 DEBUG [StoreCloser-TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/B/111429484d764ec88cbb39f48033246d to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/B/111429484d764ec88cbb39f48033246d 2024-11-18T20:13:00,884 DEBUG [StoreCloser-TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/B/dcf9fbf0aa3d492bb3146b6642e4f833 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/B/dcf9fbf0aa3d492bb3146b6642e4f833 2024-11-18T20:13:00,885 DEBUG [StoreCloser-TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/B/944b2989e5dd45d1abf25c5f89f1658c to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/B/944b2989e5dd45d1abf25c5f89f1658c 2024-11-18T20:13:00,885 DEBUG [StoreCloser-TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/B/bd6d8cccd2a340c8892eb9c7980aef18 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/B/bd6d8cccd2a340c8892eb9c7980aef18 2024-11-18T20:13:00,887 DEBUG [StoreCloser-TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/C/a237f2ac7f5b43be98189738aaad1223, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/C/e4072932f97e4bc9aa4ea0dab2795a1d, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/C/e1bedff458a04273817b88fcd6019fe7, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/C/5ec448785187415e8bbaa060c1064b04, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/C/d26895fa6d6040d1b7d3b944241ec251, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/C/e8c4015a7e704d1eb3191eef0cd574a9, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/C/c8b0082833a84d63aedd266b9c332cb7, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/C/0dbae9b9d64c48abba6ceca4cca5a469, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/C/ee7dc1a04dda47929cba204f921f23e7, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/C/89b3d2b5737841fbaf37bdb21fdcd126, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/C/ea1d95882f824602851c7f4e59a41594, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/C/affa94e1672a44af8bd19a93bcc96f16, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/C/927ee34d8c3e431c942bde6ee59d18ec, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/C/1314a6e353784c598c758b6208c319dc, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/C/374ee136231143aeb4b9fc08372ec6cf, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/C/97e5c5217b46428483dc1346f30b84b4, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/C/7d0afbc55b954b81a3dd35686040e709, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/C/8ae46bb8bf3b474588d6716479bf416c, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/C/d2bbddc6235345a69a5b3754bdae8661, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/C/6037b029a7dd45e18b4221f6b25b3d42, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/C/69af93cc11e44ad2949b24d01fc87e65, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/C/40a05af8f8b54e7c8697c423b5fd490b, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/C/0c89084eb8384f8aad2c1f61d7040188, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/C/36e133405d354127b3a2ff033fe4bd2a] to archive 2024-11-18T20:13:00,888 DEBUG [StoreCloser-TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-18T20:13:00,889 DEBUG [StoreCloser-TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/C/a237f2ac7f5b43be98189738aaad1223 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/C/a237f2ac7f5b43be98189738aaad1223 2024-11-18T20:13:00,890 DEBUG [StoreCloser-TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/C/e4072932f97e4bc9aa4ea0dab2795a1d to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/C/e4072932f97e4bc9aa4ea0dab2795a1d 2024-11-18T20:13:00,891 DEBUG [StoreCloser-TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/C/e1bedff458a04273817b88fcd6019fe7 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/C/e1bedff458a04273817b88fcd6019fe7 2024-11-18T20:13:00,892 DEBUG [StoreCloser-TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/C/5ec448785187415e8bbaa060c1064b04 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/C/5ec448785187415e8bbaa060c1064b04 2024-11-18T20:13:00,893 DEBUG [StoreCloser-TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/C/d26895fa6d6040d1b7d3b944241ec251 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/C/d26895fa6d6040d1b7d3b944241ec251 2024-11-18T20:13:00,894 DEBUG [StoreCloser-TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/C/e8c4015a7e704d1eb3191eef0cd574a9 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/C/e8c4015a7e704d1eb3191eef0cd574a9 2024-11-18T20:13:00,895 DEBUG [StoreCloser-TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/C/c8b0082833a84d63aedd266b9c332cb7 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/C/c8b0082833a84d63aedd266b9c332cb7 2024-11-18T20:13:00,896 DEBUG [StoreCloser-TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/C/0dbae9b9d64c48abba6ceca4cca5a469 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/C/0dbae9b9d64c48abba6ceca4cca5a469 2024-11-18T20:13:00,897 DEBUG [StoreCloser-TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/C/ee7dc1a04dda47929cba204f921f23e7 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/C/ee7dc1a04dda47929cba204f921f23e7 2024-11-18T20:13:00,898 DEBUG [StoreCloser-TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/C/89b3d2b5737841fbaf37bdb21fdcd126 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/C/89b3d2b5737841fbaf37bdb21fdcd126 2024-11-18T20:13:00,899 DEBUG [StoreCloser-TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/C/ea1d95882f824602851c7f4e59a41594 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/C/ea1d95882f824602851c7f4e59a41594 2024-11-18T20:13:00,900 DEBUG [StoreCloser-TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/C/affa94e1672a44af8bd19a93bcc96f16 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/C/affa94e1672a44af8bd19a93bcc96f16 2024-11-18T20:13:00,901 DEBUG [StoreCloser-TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/C/927ee34d8c3e431c942bde6ee59d18ec to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/C/927ee34d8c3e431c942bde6ee59d18ec 2024-11-18T20:13:00,902 DEBUG [StoreCloser-TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/C/1314a6e353784c598c758b6208c319dc to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/C/1314a6e353784c598c758b6208c319dc 2024-11-18T20:13:00,903 DEBUG [StoreCloser-TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/C/374ee136231143aeb4b9fc08372ec6cf to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/C/374ee136231143aeb4b9fc08372ec6cf 2024-11-18T20:13:00,904 DEBUG [StoreCloser-TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/C/97e5c5217b46428483dc1346f30b84b4 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/C/97e5c5217b46428483dc1346f30b84b4 2024-11-18T20:13:00,905 DEBUG [StoreCloser-TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/C/7d0afbc55b954b81a3dd35686040e709 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/C/7d0afbc55b954b81a3dd35686040e709 2024-11-18T20:13:00,906 DEBUG [StoreCloser-TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/C/8ae46bb8bf3b474588d6716479bf416c to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/C/8ae46bb8bf3b474588d6716479bf416c 2024-11-18T20:13:00,907 DEBUG [StoreCloser-TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/C/d2bbddc6235345a69a5b3754bdae8661 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/C/d2bbddc6235345a69a5b3754bdae8661 2024-11-18T20:13:00,908 DEBUG [StoreCloser-TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/C/6037b029a7dd45e18b4221f6b25b3d42 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/C/6037b029a7dd45e18b4221f6b25b3d42 2024-11-18T20:13:00,909 DEBUG [StoreCloser-TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/C/69af93cc11e44ad2949b24d01fc87e65 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/C/69af93cc11e44ad2949b24d01fc87e65 2024-11-18T20:13:00,910 DEBUG [StoreCloser-TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/C/40a05af8f8b54e7c8697c423b5fd490b to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/C/40a05af8f8b54e7c8697c423b5fd490b 2024-11-18T20:13:00,911 DEBUG [StoreCloser-TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/C/0c89084eb8384f8aad2c1f61d7040188 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/C/0c89084eb8384f8aad2c1f61d7040188 2024-11-18T20:13:00,912 DEBUG [StoreCloser-TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/C/36e133405d354127b3a2ff033fe4bd2a to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/C/36e133405d354127b3a2ff033fe4bd2a 2024-11-18T20:13:00,916 DEBUG [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/recovered.edits/355.seqid, newMaxSeqId=355, maxSeqId=4 2024-11-18T20:13:00,917 INFO [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8. 2024-11-18T20:13:00,917 DEBUG [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HRegion(1676): Region close journal for 9d5f6ceb1ccdf5b797c44f37788d88e8: Waiting for close lock at 1731960779145Running coprocessor pre-close hooks at 1731960779145Disabling compacts and flushes for region at 1731960779145Disabling writes for close at 1731960779145Obtaining lock to block concurrent updates at 1731960779145Preparing flush snapshotting stores in 9d5f6ceb1ccdf5b797c44f37788d88e8 at 1731960779145Finished memstore snapshotting TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8., syncing WAL and waiting on mvcc, flushsize=dataSize=27480, getHeapSize=72720, getOffHeapSize=0, getCellsCount=600 at 1731960779146 (+1 ms)Flushing stores of TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8. at 1731960779146Flushing 9d5f6ceb1ccdf5b797c44f37788d88e8/A: creating writer at 1731960779146Flushing mob file 9d5f6ceb1ccdf5b797c44f37788d88e8/A: appending metadata at 1731960779152 (+6 ms)Flushing mob file 9d5f6ceb1ccdf5b797c44f37788d88e8/A: closing flushed file at 1731960779152Flushing 9d5f6ceb1ccdf5b797c44f37788d88e8/A: appending metadata at 1731960779569 (+417 ms)Flushing 9d5f6ceb1ccdf5b797c44f37788d88e8/A: closing flushed file at 1731960779569Flushing 9d5f6ceb1ccdf5b797c44f37788d88e8/B: creating writer at 1731960779984 (+415 ms)Flushing 9d5f6ceb1ccdf5b797c44f37788d88e8/B: appending metadata at 1731960779987 (+3 ms)Flushing 9d5f6ceb1ccdf5b797c44f37788d88e8/B: closing flushed file at 1731960779987Flushing 9d5f6ceb1ccdf5b797c44f37788d88e8/C: creating writer at 1731960780400 (+413 ms)Flushing 9d5f6ceb1ccdf5b797c44f37788d88e8/C: appending metadata at 1731960780403 (+3 ms)Flushing 9d5f6ceb1ccdf5b797c44f37788d88e8/C: closing flushed file at 1731960780404 (+1 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3bab3e36: reopening flushed file at 1731960780814 (+410 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6dcd7d5f: reopening flushed file at 1731960780820 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3da56b78: reopening flushed file at 1731960780826 (+6 ms)Finished flush of dataSize ~26.84 KB/27480, heapSize ~71.02 KB/72720, currentSize=0 B/0 for 9d5f6ceb1ccdf5b797c44f37788d88e8 in 1687ms, sequenceid=352, compaction requested=false at 1731960780832 (+6 ms)Writing region close event to WAL at 1731960780913 (+81 ms)Running coprocessor post-close hooks at 1731960780916 (+3 ms)Closed at 1731960780917 (+1 ms) 2024-11-18T20:13:00,919 INFO [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] handler.UnassignRegionHandler(157): Closed 9d5f6ceb1ccdf5b797c44f37788d88e8 2024-11-18T20:13:00,920 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=54 updating hbase:meta row=9d5f6ceb1ccdf5b797c44f37788d88e8, regionState=CLOSED 2024-11-18T20:13:00,921 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=55, ppid=54, state=RUNNABLE, hasLock=false; CloseRegionProcedure 9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 because future has completed 2024-11-18T20:13:00,923 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=55, resume processing ppid=54 2024-11-18T20:13:00,923 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=55, ppid=54, state=SUCCESS, hasLock=false; CloseRegionProcedure 9d5f6ceb1ccdf5b797c44f37788d88e8, server=c0a89b2656d4,35963,1731960718223 in 1.9300 sec 2024-11-18T20:13:00,925 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=54, resume processing ppid=53 2024-11-18T20:13:00,925 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=54, ppid=53, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=9d5f6ceb1ccdf5b797c44f37788d88e8, UNASSIGN in 1.9330 sec 2024-11-18T20:13:00,927 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=53, resume processing ppid=52 2024-11-18T20:13:00,927 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=53, ppid=52, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.9380 sec 2024-11-18T20:13:00,928 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731960780928"}]},"ts":"1731960780928"} 2024-11-18T20:13:00,930 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-18T20:13:00,930 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-18T20:13:00,932 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=52, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.9490 sec 2024-11-18T20:13:01,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-18T20:13:01,133 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-18T20:13:01,134 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-18T20:13:01,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] procedure2.ProcedureExecutor(1139): Stored pid=56, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-18T20:13:01,135 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=56, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-18T20:13:01,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-18T20:13:01,136 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=56, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-18T20:13:01,139 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8 2024-11-18T20:13:01,142 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/A, FileablePath, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/B, FileablePath, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/C, FileablePath, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/recovered.edits] 2024-11-18T20:13:01,146 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/A/7a4c9c2b5da54ed6a716bd200c941207 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/A/7a4c9c2b5da54ed6a716bd200c941207 2024-11-18T20:13:01,148 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/A/9d1545bcc0054d9eae892e890f8c0ae8 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/A/9d1545bcc0054d9eae892e890f8c0ae8 2024-11-18T20:13:01,152 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/B/0c51cf40051b4fa9b5ea10dded4e1f43 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/B/0c51cf40051b4fa9b5ea10dded4e1f43 2024-11-18T20:13:01,153 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/B/44c0d64d5bf94a3da79a366c4492faba to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/B/44c0d64d5bf94a3da79a366c4492faba 2024-11-18T20:13:01,157 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/C/3419e63ff3af43cd9201eee2f8fedbd1 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/C/3419e63ff3af43cd9201eee2f8fedbd1 2024-11-18T20:13:01,158 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/C/d7d5f1184d8e4d23aa178efdc3682c6f to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/C/d7d5f1184d8e4d23aa178efdc3682c6f 2024-11-18T20:13:01,160 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/recovered.edits/355.seqid to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8/recovered.edits/355.seqid 2024-11-18T20:13:01,161 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/9d5f6ceb1ccdf5b797c44f37788d88e8 2024-11-18T20:13:01,161 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-18T20:13:01,162 DEBUG [PEWorker-2 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-18T20:13:01,162 DEBUG [PEWorker-2 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-18T20:13:01,168 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411181fb7c2e57ab142aab585e683013e1eb1_9d5f6ceb1ccdf5b797c44f37788d88e8 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411181fb7c2e57ab142aab585e683013e1eb1_9d5f6ceb1ccdf5b797c44f37788d88e8 2024-11-18T20:13:01,169 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411183d4d9ff5f7e94c7f9b9268955608bf0e_9d5f6ceb1ccdf5b797c44f37788d88e8 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411183d4d9ff5f7e94c7f9b9268955608bf0e_9d5f6ceb1ccdf5b797c44f37788d88e8 2024-11-18T20:13:01,171 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111840b857ea9a5648e688c9784e57a8e2f4_9d5f6ceb1ccdf5b797c44f37788d88e8 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111840b857ea9a5648e688c9784e57a8e2f4_9d5f6ceb1ccdf5b797c44f37788d88e8 2024-11-18T20:13:01,172 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111843d874b4b648487d9fb850d89d68d699_9d5f6ceb1ccdf5b797c44f37788d88e8 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111843d874b4b648487d9fb850d89d68d699_9d5f6ceb1ccdf5b797c44f37788d88e8 2024-11-18T20:13:01,173 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118480747fc42754e479a9faf4574c6013a_9d5f6ceb1ccdf5b797c44f37788d88e8 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118480747fc42754e479a9faf4574c6013a_9d5f6ceb1ccdf5b797c44f37788d88e8 2024-11-18T20:13:01,177 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411184e7fdd3d0c0f4592b9281e85c5d73302_9d5f6ceb1ccdf5b797c44f37788d88e8 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411184e7fdd3d0c0f4592b9281e85c5d73302_9d5f6ceb1ccdf5b797c44f37788d88e8 2024-11-18T20:13:01,179 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411184e9c6b4033d5409d88759cf9ffbfa09b_9d5f6ceb1ccdf5b797c44f37788d88e8 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411184e9c6b4033d5409d88759cf9ffbfa09b_9d5f6ceb1ccdf5b797c44f37788d88e8 2024-11-18T20:13:01,180 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411185c4f3aa4378a45f2bd643fec3e2e0412_9d5f6ceb1ccdf5b797c44f37788d88e8 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411185c4f3aa4378a45f2bd643fec3e2e0412_9d5f6ceb1ccdf5b797c44f37788d88e8 2024-11-18T20:13:01,183 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111875e5410bf48b41199a33e2f1d2853f1e_9d5f6ceb1ccdf5b797c44f37788d88e8 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111875e5410bf48b41199a33e2f1d2853f1e_9d5f6ceb1ccdf5b797c44f37788d88e8 2024-11-18T20:13:01,185 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411189de5b8dd2869434f91d4a501b1541fbe_9d5f6ceb1ccdf5b797c44f37788d88e8 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411189de5b8dd2869434f91d4a501b1541fbe_9d5f6ceb1ccdf5b797c44f37788d88e8 2024-11-18T20:13:01,187 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118a149197026d64f20a94f1e136e5b39c6_9d5f6ceb1ccdf5b797c44f37788d88e8 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118a149197026d64f20a94f1e136e5b39c6_9d5f6ceb1ccdf5b797c44f37788d88e8 2024-11-18T20:13:01,190 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118b0a3195e45464f848230dde07269f7b8_9d5f6ceb1ccdf5b797c44f37788d88e8 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118b0a3195e45464f848230dde07269f7b8_9d5f6ceb1ccdf5b797c44f37788d88e8 2024-11-18T20:13:01,192 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118b359682701214fa79b9c8fb5d4626a8e_9d5f6ceb1ccdf5b797c44f37788d88e8 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118b359682701214fa79b9c8fb5d4626a8e_9d5f6ceb1ccdf5b797c44f37788d88e8 2024-11-18T20:13:01,194 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118bb346766f8064b62b611b4fb3f31e19f_9d5f6ceb1ccdf5b797c44f37788d88e8 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118bb346766f8064b62b611b4fb3f31e19f_9d5f6ceb1ccdf5b797c44f37788d88e8 2024-11-18T20:13:01,195 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118ce67afc2c80c423a91e2193366e4f2b9_9d5f6ceb1ccdf5b797c44f37788d88e8 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118ce67afc2c80c423a91e2193366e4f2b9_9d5f6ceb1ccdf5b797c44f37788d88e8 2024-11-18T20:13:01,196 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118f7752a19b8d14e7badba705d75c56465_9d5f6ceb1ccdf5b797c44f37788d88e8 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118f7752a19b8d14e7badba705d75c56465_9d5f6ceb1ccdf5b797c44f37788d88e8 2024-11-18T20:13:01,197 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118f80ff673c5344be2b5497315e3334ff6_9d5f6ceb1ccdf5b797c44f37788d88e8 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118f80ff673c5344be2b5497315e3334ff6_9d5f6ceb1ccdf5b797c44f37788d88e8 2024-11-18T20:13:01,199 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118fb1706e21572435b8df5edd2e6929de1_9d5f6ceb1ccdf5b797c44f37788d88e8 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118fb1706e21572435b8df5edd2e6929de1_9d5f6ceb1ccdf5b797c44f37788d88e8 2024-11-18T20:13:01,200 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118fcd9a6c5a19f46ef8039b75339201f30_9d5f6ceb1ccdf5b797c44f37788d88e8 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118fcd9a6c5a19f46ef8039b75339201f30_9d5f6ceb1ccdf5b797c44f37788d88e8 2024-11-18T20:13:01,200 DEBUG [PEWorker-2 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-18T20:13:01,203 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=56, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-18T20:13:01,206 WARN [PEWorker-2 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-18T20:13:01,208 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-18T20:13:01,209 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=56, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-18T20:13:01,209 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-18T20:13:01,210 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731960781209"}]},"ts":"9223372036854775807"} 2024-11-18T20:13:01,213 INFO [PEWorker-2 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-18T20:13:01,214 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 9d5f6ceb1ccdf5b797c44f37788d88e8, NAME => 'TestAcidGuarantees,,1731960747355.9d5f6ceb1ccdf5b797c44f37788d88e8.', STARTKEY => '', ENDKEY => ''}] 2024-11-18T20:13:01,214 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-18T20:13:01,214 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731960781214"}]},"ts":"9223372036854775807"} 2024-11-18T20:13:01,217 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-18T20:13:01,218 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(133): Finished pid=56, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-18T20:13:01,221 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=56, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 84 msec 2024-11-18T20:13:01,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-18T20:13:01,244 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-18T20:13:01,244 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-18T20:13:01,252 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testMobMixedAtomicity Thread=237 (was 238), OpenFileDescriptor=453 (was 455), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=478 (was 398) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=2719 (was 3402) 2024-11-18T20:13:01,263 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testGetAtomicity Thread=237, OpenFileDescriptor=453, MaxFileDescriptor=1048576, SystemLoadAverage=478, ProcessCount=11, AvailableMemoryMB=2718 2024-11-18T20:13:01,264 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-18T20:13:01,265 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-18T20:13:01,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] procedure2.ProcedureExecutor(1139): Stored pid=57, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-18T20:13:01,267 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=57, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-18T20:13:01,267 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:01,267 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 57 2024-11-18T20:13:01,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=57 2024-11-18T20:13:01,268 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=57, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-18T20:13:01,275 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742063_1239 (size=959) 2024-11-18T20:13:01,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=57 2024-11-18T20:13:01,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=57 2024-11-18T20:13:01,678 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 294b8f086325976977089df95da38d70, NAME => 'TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123 2024-11-18T20:13:01,687 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742064_1240 (size=53) 2024-11-18T20:13:01,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=57 2024-11-18T20:13:02,089 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T20:13:02,090 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 294b8f086325976977089df95da38d70, disabling compactions & flushes 2024-11-18T20:13:02,090 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70. 2024-11-18T20:13:02,090 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70. 2024-11-18T20:13:02,090 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70. after waiting 0 ms 2024-11-18T20:13:02,090 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70. 2024-11-18T20:13:02,090 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70. 2024-11-18T20:13:02,090 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 294b8f086325976977089df95da38d70: Waiting for close lock at 1731960782089Disabling compacts and flushes for region at 1731960782090 (+1 ms)Disabling writes for close at 1731960782090Writing region close event to WAL at 1731960782090Closed at 1731960782090 2024-11-18T20:13:02,092 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=57, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-18T20:13:02,092 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731960782092"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731960782092"}]},"ts":"1731960782092"} 2024-11-18T20:13:02,095 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-18T20:13:02,097 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=57, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-18T20:13:02,097 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731960782097"}]},"ts":"1731960782097"} 2024-11-18T20:13:02,098 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-18T20:13:02,099 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=58, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=294b8f086325976977089df95da38d70, ASSIGN}] 2024-11-18T20:13:02,100 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=58, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=294b8f086325976977089df95da38d70, ASSIGN 2024-11-18T20:13:02,101 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=58, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=294b8f086325976977089df95da38d70, ASSIGN; state=OFFLINE, location=c0a89b2656d4,35963,1731960718223; forceNewPlan=false, retain=false 2024-11-18T20:13:02,252 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=58 updating hbase:meta row=294b8f086325976977089df95da38d70, regionState=OPENING, regionLocation=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:02,253 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=58, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=294b8f086325976977089df95da38d70, ASSIGN because future has completed 2024-11-18T20:13:02,253 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=59, ppid=58, state=RUNNABLE, hasLock=false; OpenRegionProcedure 294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223}] 2024-11-18T20:13:02,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=57 2024-11-18T20:13:02,408 INFO [RS_OPEN_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70. 2024-11-18T20:13:02,409 DEBUG [RS_OPEN_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(7752): Opening region: {ENCODED => 294b8f086325976977089df95da38d70, NAME => 'TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70.', STARTKEY => '', ENDKEY => ''} 2024-11-18T20:13:02,409 DEBUG [RS_OPEN_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 294b8f086325976977089df95da38d70 2024-11-18T20:13:02,409 DEBUG [RS_OPEN_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T20:13:02,409 DEBUG [RS_OPEN_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(7794): checking encryption for 294b8f086325976977089df95da38d70 2024-11-18T20:13:02,409 DEBUG [RS_OPEN_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(7797): checking classloading for 294b8f086325976977089df95da38d70 2024-11-18T20:13:02,411 INFO [StoreOpener-294b8f086325976977089df95da38d70-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 294b8f086325976977089df95da38d70 2024-11-18T20:13:02,412 INFO [StoreOpener-294b8f086325976977089df95da38d70-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 294b8f086325976977089df95da38d70 columnFamilyName A 2024-11-18T20:13:02,412 DEBUG [StoreOpener-294b8f086325976977089df95da38d70-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:02,413 INFO [StoreOpener-294b8f086325976977089df95da38d70-1 {}] regionserver.HStore(327): Store=294b8f086325976977089df95da38d70/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T20:13:02,413 INFO [StoreOpener-294b8f086325976977089df95da38d70-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 294b8f086325976977089df95da38d70 2024-11-18T20:13:02,415 INFO [StoreOpener-294b8f086325976977089df95da38d70-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 294b8f086325976977089df95da38d70 columnFamilyName B 2024-11-18T20:13:02,415 DEBUG [StoreOpener-294b8f086325976977089df95da38d70-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:02,415 INFO [StoreOpener-294b8f086325976977089df95da38d70-1 {}] regionserver.HStore(327): Store=294b8f086325976977089df95da38d70/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T20:13:02,415 INFO [StoreOpener-294b8f086325976977089df95da38d70-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 294b8f086325976977089df95da38d70 2024-11-18T20:13:02,417 INFO [StoreOpener-294b8f086325976977089df95da38d70-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 294b8f086325976977089df95da38d70 columnFamilyName C 2024-11-18T20:13:02,417 DEBUG [StoreOpener-294b8f086325976977089df95da38d70-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:02,417 INFO [StoreOpener-294b8f086325976977089df95da38d70-1 {}] regionserver.HStore(327): Store=294b8f086325976977089df95da38d70/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T20:13:02,418 DEBUG [RS_OPEN_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(1038): replaying wal for 294b8f086325976977089df95da38d70 2024-11-18T20:13:02,418 DEBUG [RS_OPEN_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70 2024-11-18T20:13:02,419 DEBUG [RS_OPEN_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70 2024-11-18T20:13:02,420 DEBUG [RS_OPEN_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(1048): stopping wal replay for 294b8f086325976977089df95da38d70 2024-11-18T20:13:02,420 DEBUG [RS_OPEN_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(1060): Cleaning up temporary data for 294b8f086325976977089df95da38d70 2024-11-18T20:13:02,421 DEBUG [RS_OPEN_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-18T20:13:02,423 DEBUG [RS_OPEN_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(1093): writing seq id for 294b8f086325976977089df95da38d70 2024-11-18T20:13:02,426 DEBUG [RS_OPEN_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-18T20:13:02,426 INFO [RS_OPEN_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(1114): Opened 294b8f086325976977089df95da38d70; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71644275, jitterRate=0.06758289039134979}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-18T20:13:02,426 DEBUG [RS_OPEN_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 294b8f086325976977089df95da38d70 2024-11-18T20:13:02,427 DEBUG [RS_OPEN_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(1006): Region open journal for 294b8f086325976977089df95da38d70: Running coprocessor pre-open hook at 1731960782409Writing region info on filesystem at 1731960782409Initializing all the Stores at 1731960782410 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731960782410Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731960782411 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731960782411Cleaning up temporary data from old regions at 1731960782420 (+9 ms)Running coprocessor post-open hooks at 1731960782426 (+6 ms)Region opened successfully at 1731960782427 (+1 ms) 2024-11-18T20:13:02,427 INFO [RS_OPEN_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., pid=59, masterSystemTime=1731960782405 2024-11-18T20:13:02,429 DEBUG [RS_OPEN_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70. 2024-11-18T20:13:02,429 INFO [RS_OPEN_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70. 2024-11-18T20:13:02,430 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=58 updating hbase:meta row=294b8f086325976977089df95da38d70, regionState=OPEN, openSeqNum=2, regionLocation=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:02,431 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=59, ppid=58, state=RUNNABLE, hasLock=false; OpenRegionProcedure 294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 because future has completed 2024-11-18T20:13:02,434 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=59, resume processing ppid=58 2024-11-18T20:13:02,434 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=59, ppid=58, state=SUCCESS, hasLock=false; OpenRegionProcedure 294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 in 179 msec 2024-11-18T20:13:02,436 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=58, resume processing ppid=57 2024-11-18T20:13:02,436 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=58, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=294b8f086325976977089df95da38d70, ASSIGN in 335 msec 2024-11-18T20:13:02,436 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=57, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-18T20:13:02,437 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731960782436"}]},"ts":"1731960782436"} 2024-11-18T20:13:02,438 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-18T20:13:02,439 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=57, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-18T20:13:02,440 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=57, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1740 sec 2024-11-18T20:13:03,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=57 2024-11-18T20:13:03,413 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-18T20:13:03,414 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@78af2e00, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:13:03,414 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c0a89b2656d4,41277,-1 for getting cluster id 2024-11-18T20:13:03,415 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T20:13:03,416 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ac99df6b-7791-4693-a81a-6a5a39504490' 2024-11-18T20:13:03,416 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T20:13:03,416 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ac99df6b-7791-4693-a81a-6a5a39504490" 2024-11-18T20:13:03,416 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3a1e8e5d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:13:03,416 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c0a89b2656d4,41277,-1] 2024-11-18T20:13:03,417 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T20:13:03,417 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:13:03,418 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49962, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T20:13:03,418 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@740b78d5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:13:03,419 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T20:13:03,419 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c0a89b2656d4,35963,1731960718223, seqNum=-1] 2024-11-18T20:13:03,420 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T20:13:03,421 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54530, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T20:13:03,421 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-18T20:13:03,422 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncConnectionImpl(321): The fetched master address is c0a89b2656d4,41277,1731960717534 2024-11-18T20:13:03,422 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@64303841 2024-11-18T20:13:03,422 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-18T20:13:03,423 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49974, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-18T20:13:03,425 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1d8da64d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:13:03,425 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c0a89b2656d4,41277,-1 for getting cluster id 2024-11-18T20:13:03,425 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T20:13:03,425 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ac99df6b-7791-4693-a81a-6a5a39504490' 2024-11-18T20:13:03,426 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T20:13:03,426 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ac99df6b-7791-4693-a81a-6a5a39504490" 2024-11-18T20:13:03,426 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@49274cf1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:13:03,426 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c0a89b2656d4,41277,-1] 2024-11-18T20:13:03,426 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T20:13:03,426 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:13:03,427 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50002, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T20:13:03,428 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7dbd7b76, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:13:03,428 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@50502613, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:13:03,429 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c0a89b2656d4,41277,-1 for getting cluster id 2024-11-18T20:13:03,429 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T20:13:03,429 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ac99df6b-7791-4693-a81a-6a5a39504490' 2024-11-18T20:13:03,430 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T20:13:03,430 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ac99df6b-7791-4693-a81a-6a5a39504490" 2024-11-18T20:13:03,430 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@491cf695, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:13:03,430 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c0a89b2656d4,41277,-1] 2024-11-18T20:13:03,430 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T20:13:03,430 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:13:03,431 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50024, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T20:13:03,432 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7e2e4163, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:13:03,432 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@554689fb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:13:03,432 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c0a89b2656d4,41277,-1 for getting cluster id 2024-11-18T20:13:03,432 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T20:13:03,433 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ac99df6b-7791-4693-a81a-6a5a39504490' 2024-11-18T20:13:03,433 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T20:13:03,433 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ac99df6b-7791-4693-a81a-6a5a39504490" 2024-11-18T20:13:03,433 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@78f761e7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:13:03,434 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c0a89b2656d4,41277,-1] 2024-11-18T20:13:03,434 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T20:13:03,434 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:13:03,434 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50032, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T20:13:03,435 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1f43e70b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:13:03,436 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@72325ab8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:13:03,436 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c0a89b2656d4,41277,-1 for getting cluster id 2024-11-18T20:13:03,436 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T20:13:03,436 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ac99df6b-7791-4693-a81a-6a5a39504490' 2024-11-18T20:13:03,437 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T20:13:03,437 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ac99df6b-7791-4693-a81a-6a5a39504490" 2024-11-18T20:13:03,437 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4674ed98, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:13:03,437 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c0a89b2656d4,41277,-1] 2024-11-18T20:13:03,437 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T20:13:03,437 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:13:03,438 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50038, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T20:13:03,439 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4c29424, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:13:03,439 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6686bc48, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:13:03,439 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c0a89b2656d4,41277,-1 for getting cluster id 2024-11-18T20:13:03,439 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T20:13:03,440 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ac99df6b-7791-4693-a81a-6a5a39504490' 2024-11-18T20:13:03,440 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T20:13:03,440 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ac99df6b-7791-4693-a81a-6a5a39504490" 2024-11-18T20:13:03,441 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@799a9019, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:13:03,441 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c0a89b2656d4,41277,-1] 2024-11-18T20:13:03,441 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T20:13:03,441 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:13:03,442 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50052, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T20:13:03,442 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@684a22b8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:13:03,443 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@47f17e9d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:13:03,443 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c0a89b2656d4,41277,-1 for getting cluster id 2024-11-18T20:13:03,443 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T20:13:03,444 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ac99df6b-7791-4693-a81a-6a5a39504490' 2024-11-18T20:13:03,444 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T20:13:03,444 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ac99df6b-7791-4693-a81a-6a5a39504490" 2024-11-18T20:13:03,444 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3f1750d1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:13:03,444 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c0a89b2656d4,41277,-1] 2024-11-18T20:13:03,444 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T20:13:03,445 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:13:03,445 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50070, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T20:13:03,446 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7394d9f2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:13:03,447 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@675e61a9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:13:03,447 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c0a89b2656d4,41277,-1 for getting cluster id 2024-11-18T20:13:03,447 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T20:13:03,448 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ac99df6b-7791-4693-a81a-6a5a39504490' 2024-11-18T20:13:03,448 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T20:13:03,448 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ac99df6b-7791-4693-a81a-6a5a39504490" 2024-11-18T20:13:03,448 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@13ee1210, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:13:03,448 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c0a89b2656d4,41277,-1] 2024-11-18T20:13:03,448 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T20:13:03,449 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:13:03,449 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50086, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T20:13:03,450 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@12845ba3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:13:03,451 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4b9635b3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:13:03,451 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c0a89b2656d4,41277,-1 for getting cluster id 2024-11-18T20:13:03,451 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T20:13:03,452 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ac99df6b-7791-4693-a81a-6a5a39504490' 2024-11-18T20:13:03,452 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T20:13:03,452 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ac99df6b-7791-4693-a81a-6a5a39504490" 2024-11-18T20:13:03,452 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@14e6aaf1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:13:03,452 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c0a89b2656d4,41277,-1] 2024-11-18T20:13:03,452 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T20:13:03,452 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:13:03,453 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50098, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T20:13:03,453 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3e170bb7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:13:03,454 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2611025, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:13:03,454 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c0a89b2656d4,41277,-1 for getting cluster id 2024-11-18T20:13:03,454 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T20:13:03,455 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ac99df6b-7791-4693-a81a-6a5a39504490' 2024-11-18T20:13:03,455 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T20:13:03,455 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ac99df6b-7791-4693-a81a-6a5a39504490" 2024-11-18T20:13:03,455 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@153c2ac, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:13:03,455 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c0a89b2656d4,41277,-1] 2024-11-18T20:13:03,455 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T20:13:03,455 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:13:03,456 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50122, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T20:13:03,456 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2d9c570a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:13:03,457 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@8a8ae50, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:13:03,457 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c0a89b2656d4,41277,-1 for getting cluster id 2024-11-18T20:13:03,457 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T20:13:03,458 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ac99df6b-7791-4693-a81a-6a5a39504490' 2024-11-18T20:13:03,458 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T20:13:03,458 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ac99df6b-7791-4693-a81a-6a5a39504490" 2024-11-18T20:13:03,458 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@63a919e7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:13:03,458 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c0a89b2656d4,41277,-1] 2024-11-18T20:13:03,459 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T20:13:03,459 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:13:03,459 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50144, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T20:13:03,460 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@69330038, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:13:03,461 DEBUG [Thread-1125 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T20:13:03,461 DEBUG [Thread-1130 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T20:13:03,461 DEBUG [Thread-1134 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T20:13:03,461 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c0a89b2656d4,35963,1731960718223, seqNum=-1] 2024-11-18T20:13:03,461 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c0a89b2656d4,35963,1731960718223, seqNum=-1] 2024-11-18T20:13:03,461 DEBUG [Thread-1132 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T20:13:03,462 DEBUG [Thread-1121 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T20:13:03,462 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T20:13:03,462 DEBUG [Thread-1136 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T20:13:03,462 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c0a89b2656d4,35963,1731960718223, seqNum=-1] 2024-11-18T20:13:03,462 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T20:13:03,462 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c0a89b2656d4,35963,1731960718223, seqNum=-1] 2024-11-18T20:13:03,462 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T20:13:03,462 DEBUG [Thread-1138 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T20:13:03,462 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T20:13:03,462 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c0a89b2656d4,35963,1731960718223, seqNum=-1] 2024-11-18T20:13:03,463 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T20:13:03,463 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c0a89b2656d4,35963,1731960718223, seqNum=-1] 2024-11-18T20:13:03,463 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T20:13:03,463 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54532, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T20:13:03,463 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54562, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T20:13:03,463 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54546, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T20:13:03,464 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c0a89b2656d4,35963,1731960718223, seqNum=-1] 2024-11-18T20:13:03,464 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54566, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T20:13:03,464 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T20:13:03,464 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54570, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T20:13:03,464 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54582, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T20:13:03,465 DEBUG [Thread-1119 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T20:13:03,465 DEBUG [Thread-1123 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T20:13:03,465 DEBUG [Thread-1127 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T20:13:03,465 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54594, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T20:13:03,466 DEBUG [Thread-1123 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c0a89b2656d4,35963,1731960718223, seqNum=-1] 2024-11-18T20:13:03,466 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2] 2024-11-18T20:13:03,466 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2] 2024-11-18T20:13:03,466 DEBUG [Thread-1127 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c0a89b2656d4,35963,1731960718223, seqNum=-1] 2024-11-18T20:13:03,466 DEBUG [Thread-1123 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T20:13:03,466 DEBUG [Thread-1127 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T20:13:03,467 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c0a89b2656d4,35963,1731960718223, seqNum=-1] 2024-11-18T20:13:03,467 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T20:13:03,468 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54602, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T20:13:03,468 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54600, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T20:13:03,468 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54614, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T20:13:03,468 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2] 2024-11-18T20:13:03,468 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2] 2024-11-18T20:13:03,468 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2] 2024-11-18T20:13:03,469 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2] 2024-11-18T20:13:03,469 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2] 2024-11-18T20:13:03,470 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-18T20:13:03,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] procedure2.ProcedureExecutor(1139): Stored pid=60, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=60, table=TestAcidGuarantees 2024-11-18T20:13:03,472 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2] 2024-11-18T20:13:03,472 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=60, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=60, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-18T20:13:03,472 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2] 2024-11-18T20:13:03,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-18T20:13:03,473 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2] 2024-11-18T20:13:03,474 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=60, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=60, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-18T20:13:03,474 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=61, ppid=60, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-18T20:13:03,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(8855): Flush requested on 294b8f086325976977089df95da38d70 2024-11-18T20:13:03,490 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-18T20:13:03,490 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 294b8f086325976977089df95da38d70 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-18T20:13:03,535 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/.tmp/A/2f328b8a126d4d1a85c43d2f16bb4d46 is 50, key is test_row_0/A:col10/1731960783482/Put/seqid=0 2024-11-18T20:13:03,555 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:03,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54570 deadline: 1731960843545, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:03,556 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:03,556 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:03,556 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:13:03,557 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:03,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54602 deadline: 1731960843549, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:03,557 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:03,557 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:03,557 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:03,557 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:13:03,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54600 deadline: 1731960843549, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:03,558 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:03,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54532 deadline: 1731960843545, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:03,558 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:03,558 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:03,558 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:03,558 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:13:03,558 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:03,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54614 deadline: 1731960843551, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:03,558 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:03,558 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:13:03,558 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:03,558 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:03,558 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:13:03,583 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742065_1241 (size=16681) 2024-11-18T20:13:03,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-18T20:13:03,629 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35963 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-11-18T20:13:03,630 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70. 2024-11-18T20:13:03,630 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70. as already flushing 2024-11-18T20:13:03,630 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70. 2024-11-18T20:13:03,630 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => 294b8f086325976977089df95da38d70, NAME => 'TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:13:03,630 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => 294b8f086325976977089df95da38d70, NAME => 'TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:13:03,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 294b8f086325976977089df95da38d70, NAME => 'TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 294b8f086325976977089df95da38d70, NAME => 'TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:13:03,667 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:03,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54570 deadline: 1731960843663, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:03,668 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:03,668 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:03,668 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:03,668 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:13:03,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54600 deadline: 1731960843663, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:03,668 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:03,668 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:03,668 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:03,668 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:13:03,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54532 deadline: 1731960843664, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:03,669 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:03,669 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:03,669 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:03,669 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:13:03,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54602 deadline: 1731960843664, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:03,669 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:03,669 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:03,669 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:03,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54614 deadline: 1731960843664, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:03,669 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:13:03,670 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:03,670 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:03,670 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:13:03,784 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35963 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-11-18T20:13:03,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70. 2024-11-18T20:13:03,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70. as already flushing 2024-11-18T20:13:03,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70. 2024-11-18T20:13:03,784 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => 294b8f086325976977089df95da38d70, NAME => 'TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:13:03,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => 294b8f086325976977089df95da38d70, NAME => 'TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:13:03,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 294b8f086325976977089df95da38d70, NAME => 'TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 294b8f086325976977089df95da38d70, NAME => 'TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:13:03,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-18T20:13:03,875 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:03,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54602 deadline: 1731960843873, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:03,875 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:03,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54570 deadline: 1731960843874, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:03,875 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:03,875 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:03,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54600 deadline: 1731960843874, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:03,875 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:03,875 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:03,875 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:13:03,876 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:03,876 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:13:03,876 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:03,876 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:03,876 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:13:03,876 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:03,876 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:03,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54532 deadline: 1731960843874, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:03,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54614 deadline: 1731960843874, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:03,876 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:03,876 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:03,876 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:03,876 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:03,876 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:13:03,876 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:13:03,936 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35963 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-11-18T20:13:03,937 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70. 2024-11-18T20:13:03,937 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70. as already flushing 2024-11-18T20:13:03,937 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70. 2024-11-18T20:13:03,937 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => 294b8f086325976977089df95da38d70, NAME => 'TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:13:03,937 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => 294b8f086325976977089df95da38d70, NAME => 'TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:13:03,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 294b8f086325976977089df95da38d70, NAME => 'TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 294b8f086325976977089df95da38d70, NAME => 'TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:13:03,983 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/.tmp/A/2f328b8a126d4d1a85c43d2f16bb4d46 2024-11-18T20:13:04,023 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/.tmp/B/02051258119b47c485c2b156ef4fe198 is 50, key is test_row_0/B:col10/1731960783482/Put/seqid=0 2024-11-18T20:13:04,059 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742066_1242 (size=12001) 2024-11-18T20:13:04,060 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/.tmp/B/02051258119b47c485c2b156ef4fe198 2024-11-18T20:13:04,084 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/.tmp/C/471fbc201cb14abba98e4f22c3276257 is 50, key is test_row_0/C:col10/1731960783482/Put/seqid=0 2024-11-18T20:13:04,090 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35963 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-11-18T20:13:04,090 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70. 2024-11-18T20:13:04,090 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70. as already flushing 2024-11-18T20:13:04,090 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70. 2024-11-18T20:13:04,090 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => 294b8f086325976977089df95da38d70, NAME => 'TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:13:04,090 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => 294b8f086325976977089df95da38d70, NAME => 'TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:13:04,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 294b8f086325976977089df95da38d70, NAME => 'TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 294b8f086325976977089df95da38d70, NAME => 'TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:13:04,094 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742067_1243 (size=12001) 2024-11-18T20:13:04,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-18T20:13:04,185 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:04,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54602 deadline: 1731960844183, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:04,185 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:04,185 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:04,185 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:04,185 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:13:04,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54600 deadline: 1731960844183, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:04,186 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:04,186 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:04,186 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:04,186 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:13:04,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54614 deadline: 1731960844183, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:04,186 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:04,186 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:04,186 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:04,186 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:13:04,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54570 deadline: 1731960844183, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:04,186 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:04,186 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:04,186 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:04,187 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:13:04,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54532 deadline: 1731960844183, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:04,187 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:04,187 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:04,187 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:13:04,243 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35963 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-11-18T20:13:04,243 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70. 2024-11-18T20:13:04,243 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70. as already flushing 2024-11-18T20:13:04,243 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70. 2024-11-18T20:13:04,243 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => 294b8f086325976977089df95da38d70, NAME => 'TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:13:04,243 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => 294b8f086325976977089df95da38d70, NAME => 'TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:13:04,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 294b8f086325976977089df95da38d70, NAME => 'TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 294b8f086325976977089df95da38d70, NAME => 'TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:13:04,398 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35963 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-11-18T20:13:04,398 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70. 2024-11-18T20:13:04,399 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70. as already flushing 2024-11-18T20:13:04,399 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70. 2024-11-18T20:13:04,399 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => 294b8f086325976977089df95da38d70, NAME => 'TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:13:04,399 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => 294b8f086325976977089df95da38d70, NAME => 'TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:13:04,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 294b8f086325976977089df95da38d70, NAME => 'TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 294b8f086325976977089df95da38d70, NAME => 'TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:13:04,495 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/.tmp/C/471fbc201cb14abba98e4f22c3276257 2024-11-18T20:13:04,501 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/.tmp/A/2f328b8a126d4d1a85c43d2f16bb4d46 as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/A/2f328b8a126d4d1a85c43d2f16bb4d46 2024-11-18T20:13:04,506 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/A/2f328b8a126d4d1a85c43d2f16bb4d46, entries=250, sequenceid=15, filesize=16.3 K 2024-11-18T20:13:04,507 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/.tmp/B/02051258119b47c485c2b156ef4fe198 as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/B/02051258119b47c485c2b156ef4fe198 2024-11-18T20:13:04,512 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/B/02051258119b47c485c2b156ef4fe198, entries=150, sequenceid=15, filesize=11.7 K 2024-11-18T20:13:04,513 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/.tmp/C/471fbc201cb14abba98e4f22c3276257 as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/C/471fbc201cb14abba98e4f22c3276257 2024-11-18T20:13:04,519 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/C/471fbc201cb14abba98e4f22c3276257, entries=150, sequenceid=15, filesize=11.7 K 2024-11-18T20:13:04,520 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=134.18 KB/137400 for 294b8f086325976977089df95da38d70 in 1030ms, sequenceid=15, compaction requested=false 2024-11-18T20:13:04,520 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-18T20:13:04,521 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 294b8f086325976977089df95da38d70: 2024-11-18T20:13:04,552 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35963 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-11-18T20:13:04,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70. 2024-11-18T20:13:04,552 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2902): Flushing 294b8f086325976977089df95da38d70 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-18T20:13:04,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/.tmp/A/68b7edf47fb44fad8c46066aa8579a85 is 50, key is test_row_0/A:col10/1731960783547/Put/seqid=0 2024-11-18T20:13:04,598 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742068_1244 (size=12001) 2024-11-18T20:13:04,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-18T20:13:04,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(8855): Flush requested on 294b8f086325976977089df95da38d70 2024-11-18T20:13:04,694 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70. as already flushing 2024-11-18T20:13:04,709 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:04,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54614 deadline: 1731960844704, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:04,709 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:04,709 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:04,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54532 deadline: 1731960844705, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:04,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54600 deadline: 1731960844702, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:04,710 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:04,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54570 deadline: 1731960844705, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:04,710 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:04,710 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:04,710 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:13:04,710 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:04,710 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:04,710 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:04,710 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:13:04,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54602 deadline: 1731960844708, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:04,711 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:04,711 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:04,711 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:13:04,712 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:04,712 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:04,712 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:13:04,714 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:04,714 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:04,714 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:13:04,816 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:04,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54570 deadline: 1731960844813, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:04,816 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:04,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54600 deadline: 1731960844814, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:04,817 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:04,817 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:04,817 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:04,817 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:13:04,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54602 deadline: 1731960844814, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:04,817 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:04,817 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:04,817 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:13:04,817 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:04,817 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:04,817 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:13:04,826 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:04,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54614 deadline: 1731960844824, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:04,827 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:04,827 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:04,827 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:04,827 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:13:04,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54532 deadline: 1731960844824, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:04,827 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:04,827 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:04,827 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:13:04,999 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=38 (bloomFilter=true), to=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/.tmp/A/68b7edf47fb44fad8c46066aa8579a85 2024-11-18T20:13:05,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/.tmp/B/1356222be6be4ad089116c568b356d97 is 50, key is test_row_0/B:col10/1731960783547/Put/seqid=0 2024-11-18T20:13:05,021 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742069_1245 (size=12001) 2024-11-18T20:13:05,025 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:05,025 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:05,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54570 deadline: 1731960845023, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:05,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54600 deadline: 1731960845023, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:05,026 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:05,026 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:05,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54602 deadline: 1731960845025, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:05,026 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:05,026 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:13:05,026 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:05,026 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:05,026 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:13:05,026 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:05,026 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:05,026 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:13:05,034 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:05,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54532 deadline: 1731960845033, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:05,035 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:05,035 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:05,035 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:13:05,036 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:05,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54614 deadline: 1731960845033, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:05,037 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:05,037 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:05,037 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:13:05,334 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:05,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54600 deadline: 1731960845333, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:05,334 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:05,334 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:05,334 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:05,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54570 deadline: 1731960845333, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:05,334 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:13:05,334 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:05,334 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:05,335 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:05,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54602 deadline: 1731960845333, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:05,335 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:13:05,335 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:05,335 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:05,335 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:13:05,344 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:05,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54532 deadline: 1731960845343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:05,345 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:05,345 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:05,345 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:05,345 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:13:05,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54614 deadline: 1731960845343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:05,345 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:05,345 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:05,345 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:13:05,422 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=38 (bloomFilter=true), to=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/.tmp/B/1356222be6be4ad089116c568b356d97 2024-11-18T20:13:05,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/.tmp/C/8060693bb295482fb8078d089a7d8fe8 is 50, key is test_row_0/C:col10/1731960783547/Put/seqid=0 2024-11-18T20:13:05,440 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742070_1246 (size=12001) 2024-11-18T20:13:05,441 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=38 (bloomFilter=true), to=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/.tmp/C/8060693bb295482fb8078d089a7d8fe8 2024-11-18T20:13:05,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/.tmp/A/68b7edf47fb44fad8c46066aa8579a85 as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/A/68b7edf47fb44fad8c46066aa8579a85 2024-11-18T20:13:05,455 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/A/68b7edf47fb44fad8c46066aa8579a85, entries=150, sequenceid=38, filesize=11.7 K 2024-11-18T20:13:05,456 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/.tmp/B/1356222be6be4ad089116c568b356d97 as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/B/1356222be6be4ad089116c568b356d97 2024-11-18T20:13:05,462 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/B/1356222be6be4ad089116c568b356d97, entries=150, sequenceid=38, filesize=11.7 K 2024-11-18T20:13:05,463 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/.tmp/C/8060693bb295482fb8078d089a7d8fe8 as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/C/8060693bb295482fb8078d089a7d8fe8 2024-11-18T20:13:05,469 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/C/8060693bb295482fb8078d089a7d8fe8, entries=150, sequenceid=38, filesize=11.7 K 2024-11-18T20:13:05,470 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=73.80 KB/75570 for 294b8f086325976977089df95da38d70 in 918ms, sequenceid=38, compaction requested=false 2024-11-18T20:13:05,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2603): Flush status journal for 294b8f086325976977089df95da38d70: 2024-11-18T20:13:05,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70. 2024-11-18T20:13:05,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=61 2024-11-18T20:13:05,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.HMaster(4169): Remote procedure done, pid=61 2024-11-18T20:13:05,476 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=61, resume processing ppid=60 2024-11-18T20:13:05,476 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=61, ppid=60, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9980 sec 2024-11-18T20:13:05,479 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=60, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=60, table=TestAcidGuarantees in 2.0070 sec 2024-11-18T20:13:05,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-18T20:13:05,633 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-18T20:13:05,634 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-18T20:13:05,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] procedure2.ProcedureExecutor(1139): Stored pid=62, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees 2024-11-18T20:13:05,635 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=62, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-18T20:13:05,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-18T20:13:05,636 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=62, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-18T20:13:05,636 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=63, ppid=62, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-18T20:13:05,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-18T20:13:05,789 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35963 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-18T20:13:05,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70. 2024-11-18T20:13:05,789 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2902): Flushing 294b8f086325976977089df95da38d70 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-18T20:13:05,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/.tmp/A/3cabbf51719548c8a5de6ec4f94917fc is 50, key is test_row_0/A:col10/1731960784704/Put/seqid=0 2024-11-18T20:13:05,841 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742071_1247 (size=12001) 2024-11-18T20:13:05,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(8855): Flush requested on 294b8f086325976977089df95da38d70 2024-11-18T20:13:05,846 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70. as already flushing 2024-11-18T20:13:05,871 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:05,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54532 deadline: 1731960845866, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:05,872 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:05,872 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:05,872 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:13:05,874 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:05,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54614 deadline: 1731960845869, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:05,875 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:05,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54600 deadline: 1731960845871, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:05,875 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:05,875 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:05,875 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:13:05,875 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:05,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54602 deadline: 1731960845872, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:05,875 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:05,875 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:05,875 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:05,875 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:05,875 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:05,875 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:13:05,875 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:13:05,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54570 deadline: 1731960845872, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:05,876 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:05,876 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:05,876 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:13:05,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-18T20:13:05,985 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:05,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54532 deadline: 1731960845983, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:05,985 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:05,986 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:05,986 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:05,986 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:13:05,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54614 deadline: 1731960845983, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:05,986 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:05,986 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:05,986 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:05,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54600 deadline: 1731960845983, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:05,986 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:13:05,986 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:05,986 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:05,986 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:05,986 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:13:05,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54602 deadline: 1731960845983, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:05,986 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:05,987 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:05,986 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:05,987 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:13:05,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54570 deadline: 1731960845984, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:05,987 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:05,987 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:05,987 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:13:06,197 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:06,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54532 deadline: 1731960846193, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:06,198 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:06,198 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:06,198 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:06,198 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:13:06,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54614 deadline: 1731960846193, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:06,198 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:06,198 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:06,198 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:06,198 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:13:06,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54602 deadline: 1731960846193, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:06,199 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:06,198 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:06,199 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:06,199 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:13:06,199 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:06,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54600 deadline: 1731960846193, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:06,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54570 deadline: 1731960846193, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:06,199 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:06,199 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:06,200 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:13:06,200 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:06,200 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:06,200 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:13:06,242 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=52 (bloomFilter=true), to=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/.tmp/A/3cabbf51719548c8a5de6ec4f94917fc 2024-11-18T20:13:06,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/.tmp/B/da5441a44a1c42ee96a4403bdbff8054 is 50, key is test_row_0/B:col10/1731960784704/Put/seqid=0 2024-11-18T20:13:06,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-18T20:13:06,310 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742072_1248 (size=12001) 2024-11-18T20:13:06,314 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=52 (bloomFilter=true), to=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/.tmp/B/da5441a44a1c42ee96a4403bdbff8054 2024-11-18T20:13:06,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/.tmp/C/2ea265c0be284e1c98b7bfd0922ee4ed is 50, key is test_row_0/C:col10/1731960784704/Put/seqid=0 2024-11-18T20:13:06,342 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742073_1249 (size=12001) 2024-11-18T20:13:06,343 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=52 (bloomFilter=true), to=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/.tmp/C/2ea265c0be284e1c98b7bfd0922ee4ed 2024-11-18T20:13:06,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/.tmp/A/3cabbf51719548c8a5de6ec4f94917fc as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/A/3cabbf51719548c8a5de6ec4f94917fc 2024-11-18T20:13:06,357 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/A/3cabbf51719548c8a5de6ec4f94917fc, entries=150, sequenceid=52, filesize=11.7 K 2024-11-18T20:13:06,359 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/.tmp/B/da5441a44a1c42ee96a4403bdbff8054 as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/B/da5441a44a1c42ee96a4403bdbff8054 2024-11-18T20:13:06,373 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/B/da5441a44a1c42ee96a4403bdbff8054, entries=150, sequenceid=52, filesize=11.7 K 2024-11-18T20:13:06,375 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/.tmp/C/2ea265c0be284e1c98b7bfd0922ee4ed as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/C/2ea265c0be284e1c98b7bfd0922ee4ed 2024-11-18T20:13:06,384 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/C/2ea265c0be284e1c98b7bfd0922ee4ed, entries=150, sequenceid=52, filesize=11.7 K 2024-11-18T20:13:06,385 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for 294b8f086325976977089df95da38d70 in 596ms, sequenceid=52, compaction requested=true 2024-11-18T20:13:06,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2603): Flush status journal for 294b8f086325976977089df95da38d70: 2024-11-18T20:13:06,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70. 2024-11-18T20:13:06,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=63 2024-11-18T20:13:06,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.HMaster(4169): Remote procedure done, pid=63 2024-11-18T20:13:06,390 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=63, resume processing ppid=62 2024-11-18T20:13:06,390 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=63, ppid=62, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 751 msec 2024-11-18T20:13:06,392 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=62, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees in 756 msec 2024-11-18T20:13:06,457 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-18T20:13:06,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(8855): Flush requested on 294b8f086325976977089df95da38d70 2024-11-18T20:13:06,506 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-18T20:13:06,506 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 294b8f086325976977089df95da38d70 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-18T20:13:06,517 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/.tmp/A/88535c1b8c9e420994fb3e68ed8d74d9 is 50, key is test_row_0/A:col10/1731960785864/Put/seqid=0 2024-11-18T20:13:06,525 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:06,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54602 deadline: 1731960846518, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:06,526 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:06,526 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:06,526 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:06,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54614 deadline: 1731960846518, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:06,526 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:13:06,526 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:06,527 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:06,527 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:06,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54570 deadline: 1731960846519, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:06,527 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:13:06,527 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:06,527 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:06,527 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:06,527 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:13:06,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54600 deadline: 1731960846519, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:06,527 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:06,527 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:06,527 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:13:06,532 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:06,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54532 deadline: 1731960846526, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:06,533 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:06,533 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:06,533 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:13:06,541 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742074_1250 (size=14341) 2024-11-18T20:13:06,543 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=75 (bloomFilter=true), to=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/.tmp/A/88535c1b8c9e420994fb3e68ed8d74d9 2024-11-18T20:13:06,567 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/.tmp/B/a5f721f5cb0846dab70ce67344a0cbb1 is 50, key is test_row_0/B:col10/1731960785864/Put/seqid=0 2024-11-18T20:13:06,586 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742075_1251 (size=12001) 2024-11-18T20:13:06,589 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=75 (bloomFilter=true), to=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/.tmp/B/a5f721f5cb0846dab70ce67344a0cbb1 2024-11-18T20:13:06,599 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/.tmp/C/1ee05268c255479597687f7a1957da7a is 50, key is test_row_0/C:col10/1731960785864/Put/seqid=0 2024-11-18T20:13:06,605 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742076_1252 (size=12001) 2024-11-18T20:13:06,606 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=75 (bloomFilter=true), to=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/.tmp/C/1ee05268c255479597687f7a1957da7a 2024-11-18T20:13:06,613 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/.tmp/A/88535c1b8c9e420994fb3e68ed8d74d9 as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/A/88535c1b8c9e420994fb3e68ed8d74d9 2024-11-18T20:13:06,626 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/A/88535c1b8c9e420994fb3e68ed8d74d9, entries=200, sequenceid=75, filesize=14.0 K 2024-11-18T20:13:06,627 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/.tmp/B/a5f721f5cb0846dab70ce67344a0cbb1 as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/B/a5f721f5cb0846dab70ce67344a0cbb1 2024-11-18T20:13:06,639 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:06,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54602 deadline: 1731960846634, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:06,640 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:06,640 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:06,640 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:13:06,640 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:06,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54614 deadline: 1731960846634, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:06,640 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:06,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54570 deadline: 1731960846634, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:06,641 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:06,641 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:06,641 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:13:06,641 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:06,641 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:06,641 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:13:06,641 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:06,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54600 deadline: 1731960846634, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:06,642 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:06,642 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:06,642 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:13:06,643 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/B/a5f721f5cb0846dab70ce67344a0cbb1, entries=150, sequenceid=75, filesize=11.7 K 2024-11-18T20:13:06,644 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/.tmp/C/1ee05268c255479597687f7a1957da7a as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/C/1ee05268c255479597687f7a1957da7a 2024-11-18T20:13:06,650 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:06,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54532 deadline: 1731960846643, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:06,653 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:06,653 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:06,653 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:13:06,664 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/C/1ee05268c255479597687f7a1957da7a, entries=150, sequenceid=75, filesize=11.7 K 2024-11-18T20:13:06,665 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for 294b8f086325976977089df95da38d70 in 159ms, sequenceid=75, compaction requested=true 2024-11-18T20:13:06,665 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 294b8f086325976977089df95da38d70: 2024-11-18T20:13:06,665 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 294b8f086325976977089df95da38d70:A, priority=-2147483648, current under compaction store size is 1 2024-11-18T20:13:06,665 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:13:06,665 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-18T20:13:06,665 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 294b8f086325976977089df95da38d70:B, priority=-2147483648, current under compaction store size is 2 2024-11-18T20:13:06,665 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:13:06,665 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-18T20:13:06,665 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 294b8f086325976977089df95da38d70:C, priority=-2147483648, current under compaction store size is 3 2024-11-18T20:13:06,665 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T20:13:06,676 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 55024 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-18T20:13:06,676 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HStore(1541): 294b8f086325976977089df95da38d70/A is initiating minor compaction (all files) 2024-11-18T20:13:06,676 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 294b8f086325976977089df95da38d70/A in TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70. 2024-11-18T20:13:06,676 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/A/2f328b8a126d4d1a85c43d2f16bb4d46, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/A/68b7edf47fb44fad8c46066aa8579a85, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/A/3cabbf51719548c8a5de6ec4f94917fc, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/A/88535c1b8c9e420994fb3e68ed8d74d9] into tmpdir=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/.tmp, totalSize=53.7 K 2024-11-18T20:13:06,678 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-18T20:13:06,678 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HStore(1541): 294b8f086325976977089df95da38d70/B is initiating minor compaction (all files) 2024-11-18T20:13:06,678 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 294b8f086325976977089df95da38d70/B in TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70. 2024-11-18T20:13:06,679 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/B/02051258119b47c485c2b156ef4fe198, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/B/1356222be6be4ad089116c568b356d97, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/B/da5441a44a1c42ee96a4403bdbff8054, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/B/a5f721f5cb0846dab70ce67344a0cbb1] into tmpdir=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/.tmp, totalSize=46.9 K 2024-11-18T20:13:06,679 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2f328b8a126d4d1a85c43d2f16bb4d46, keycount=250, bloomtype=ROW, size=16.3 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1731960783475 2024-11-18T20:13:06,679 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.Compactor(225): Compacting 02051258119b47c485c2b156ef4fe198, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1731960783482 2024-11-18T20:13:06,679 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] compactions.Compactor(225): Compacting 68b7edf47fb44fad8c46066aa8579a85, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=38, earliestPutTs=1731960783532 2024-11-18T20:13:06,680 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3cabbf51719548c8a5de6ec4f94917fc, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1731960784696 2024-11-18T20:13:06,680 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.Compactor(225): Compacting 1356222be6be4ad089116c568b356d97, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=38, earliestPutTs=1731960783532 2024-11-18T20:13:06,680 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] compactions.Compactor(225): Compacting 88535c1b8c9e420994fb3e68ed8d74d9, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=75, earliestPutTs=1731960785864 2024-11-18T20:13:06,680 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.Compactor(225): Compacting da5441a44a1c42ee96a4403bdbff8054, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1731960784696 2024-11-18T20:13:06,681 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.Compactor(225): Compacting a5f721f5cb0846dab70ce67344a0cbb1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=75, earliestPutTs=1731960785864 2024-11-18T20:13:06,703 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 294b8f086325976977089df95da38d70#A#compaction#209 average throughput is 1.09 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:13:06,704 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/.tmp/A/56e41c81598b46ac93b521ec008bc0fb is 50, key is test_row_0/A:col10/1731960785864/Put/seqid=0 2024-11-18T20:13:06,712 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 294b8f086325976977089df95da38d70#B#compaction#210 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:13:06,712 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/.tmp/B/87bc89a38c1148d2859fd397687f49ae is 50, key is test_row_0/B:col10/1731960785864/Put/seqid=0 2024-11-18T20:13:06,758 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742077_1253 (size=12139) 2024-11-18T20:13:06,765 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/.tmp/A/56e41c81598b46ac93b521ec008bc0fb as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/A/56e41c81598b46ac93b521ec008bc0fb 2024-11-18T20:13:06,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-18T20:13:06,773 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-18T20:13:06,773 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 294b8f086325976977089df95da38d70/A of 294b8f086325976977089df95da38d70 into 56e41c81598b46ac93b521ec008bc0fb(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:13:06,773 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 294b8f086325976977089df95da38d70: 2024-11-18T20:13:06,773 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., storeName=294b8f086325976977089df95da38d70/A, priority=12, startTime=1731960786665; duration=0sec 2024-11-18T20:13:06,773 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T20:13:06,773 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 294b8f086325976977089df95da38d70:A 2024-11-18T20:13:06,773 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-18T20:13:06,774 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-18T20:13:06,775 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-18T20:13:06,775 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HStore(1541): 294b8f086325976977089df95da38d70/C is initiating minor compaction (all files) 2024-11-18T20:13:06,775 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 294b8f086325976977089df95da38d70/C in TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70. 2024-11-18T20:13:06,775 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/C/471fbc201cb14abba98e4f22c3276257, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/C/8060693bb295482fb8078d089a7d8fe8, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/C/2ea265c0be284e1c98b7bfd0922ee4ed, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/C/1ee05268c255479597687f7a1957da7a] into tmpdir=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/.tmp, totalSize=46.9 K 2024-11-18T20:13:06,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] procedure2.ProcedureExecutor(1139): Stored pid=64, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees 2024-11-18T20:13:06,776 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] compactions.Compactor(225): Compacting 471fbc201cb14abba98e4f22c3276257, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1731960783482 2024-11-18T20:13:06,777 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=64, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-18T20:13:06,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-18T20:13:06,778 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742078_1254 (size=12139) 2024-11-18T20:13:06,778 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8060693bb295482fb8078d089a7d8fe8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=38, earliestPutTs=1731960783532 2024-11-18T20:13:06,779 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=64, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-18T20:13:06,779 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=65, ppid=64, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-18T20:13:06,779 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2ea265c0be284e1c98b7bfd0922ee4ed, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1731960784696 2024-11-18T20:13:06,780 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1ee05268c255479597687f7a1957da7a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=75, earliestPutTs=1731960785864 2024-11-18T20:13:06,788 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/.tmp/B/87bc89a38c1148d2859fd397687f49ae as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/B/87bc89a38c1148d2859fd397687f49ae 2024-11-18T20:13:06,795 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 294b8f086325976977089df95da38d70/B of 294b8f086325976977089df95da38d70 into 87bc89a38c1148d2859fd397687f49ae(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:13:06,795 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 294b8f086325976977089df95da38d70: 2024-11-18T20:13:06,795 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., storeName=294b8f086325976977089df95da38d70/B, priority=12, startTime=1731960786665; duration=0sec 2024-11-18T20:13:06,795 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:13:06,795 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 294b8f086325976977089df95da38d70:B 2024-11-18T20:13:06,798 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 294b8f086325976977089df95da38d70#C#compaction#211 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:13:06,799 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/.tmp/C/33d7ec8003794e2a9c97e3019a47912d is 50, key is test_row_0/C:col10/1731960785864/Put/seqid=0 2024-11-18T20:13:06,848 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742079_1255 (size=12139) 2024-11-18T20:13:06,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(8855): Flush requested on 294b8f086325976977089df95da38d70 2024-11-18T20:13:06,852 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-18T20:13:06,852 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 294b8f086325976977089df95da38d70 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-11-18T20:13:06,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-18T20:13:06,903 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/.tmp/A/e4181afa412d49b98b17bbf3a2c56f2a is 50, key is test_row_0/A:col10/1731960786848/Put/seqid=0 2024-11-18T20:13:06,907 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/.tmp/C/33d7ec8003794e2a9c97e3019a47912d as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/C/33d7ec8003794e2a9c97e3019a47912d 2024-11-18T20:13:06,921 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 294b8f086325976977089df95da38d70/C of 294b8f086325976977089df95da38d70 into 33d7ec8003794e2a9c97e3019a47912d(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:13:06,921 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 294b8f086325976977089df95da38d70: 2024-11-18T20:13:06,921 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., storeName=294b8f086325976977089df95da38d70/C, priority=12, startTime=1731960786665; duration=0sec 2024-11-18T20:13:06,921 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:13:06,921 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 294b8f086325976977089df95da38d70:C 2024-11-18T20:13:06,934 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35963 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-18T20:13:06,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70. 2024-11-18T20:13:06,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70. as already flushing 2024-11-18T20:13:06,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70. 2024-11-18T20:13:06,935 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => 294b8f086325976977089df95da38d70, NAME => 'TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:13:06,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => 294b8f086325976977089df95da38d70, NAME => 'TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:13:06,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 294b8f086325976977089df95da38d70, NAME => 'TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 294b8f086325976977089df95da38d70, NAME => 'TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:13:06,950 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:06,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54602 deadline: 1731960846936, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:06,954 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:06,954 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:06,954 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:06,954 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:13:06,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54570 deadline: 1731960846942, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:06,959 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:06,959 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:06,959 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:06,959 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:13:06,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54532 deadline: 1731960846950, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:06,960 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:06,960 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:06,960 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:13:06,961 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742080_1256 (size=12001) 2024-11-18T20:13:06,960 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:06,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54614 deadline: 1731960846950, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:06,963 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:06,963 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:06,963 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:13:06,966 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:06,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54600 deadline: 1731960846951, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:06,967 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:06,967 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:06,967 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=92 (bloomFilter=true), to=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/.tmp/A/e4181afa412d49b98b17bbf3a2c56f2a 2024-11-18T20:13:06,967 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:13:06,985 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/.tmp/B/2099464fb3d9496db4e841d557d7a196 is 50, key is test_row_0/B:col10/1731960786848/Put/seqid=0 2024-11-18T20:13:07,044 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742081_1257 (size=12001) 2024-11-18T20:13:07,049 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=92 (bloomFilter=true), to=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/.tmp/B/2099464fb3d9496db4e841d557d7a196 2024-11-18T20:13:07,066 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:07,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54570 deadline: 1731960847063, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:07,067 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:07,067 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:07,067 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:13:07,067 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:07,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54602 deadline: 1731960847063, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:07,067 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:07,067 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:07,067 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:07,067 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:13:07,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54532 deadline: 1731960847063, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:07,068 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:07,068 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:07,068 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:13:07,078 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:07,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54614 deadline: 1731960847073, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:07,078 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:07,078 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:07,078 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:07,078 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:13:07,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54600 deadline: 1731960847074, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:07,079 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:07,079 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:07,079 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:13:07,087 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/.tmp/C/a48911d413104538b1b658433f9a7afe is 50, key is test_row_0/C:col10/1731960786848/Put/seqid=0 2024-11-18T20:13:07,089 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35963 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-18T20:13:07,090 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70. 2024-11-18T20:13:07,090 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70. as already flushing 2024-11-18T20:13:07,090 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70. 2024-11-18T20:13:07,090 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => 294b8f086325976977089df95da38d70, NAME => 'TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:13:07,090 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => 294b8f086325976977089df95da38d70, NAME => 'TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:13:07,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 294b8f086325976977089df95da38d70, NAME => 'TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 294b8f086325976977089df95da38d70, NAME => 'TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:13:07,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-18T20:13:07,118 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742082_1258 (size=12001) 2024-11-18T20:13:07,119 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=92 (bloomFilter=true), to=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/.tmp/C/a48911d413104538b1b658433f9a7afe 2024-11-18T20:13:07,126 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/.tmp/A/e4181afa412d49b98b17bbf3a2c56f2a as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/A/e4181afa412d49b98b17bbf3a2c56f2a 2024-11-18T20:13:07,131 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/A/e4181afa412d49b98b17bbf3a2c56f2a, entries=150, sequenceid=92, filesize=11.7 K 2024-11-18T20:13:07,133 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/.tmp/B/2099464fb3d9496db4e841d557d7a196 as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/B/2099464fb3d9496db4e841d557d7a196 2024-11-18T20:13:07,139 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/B/2099464fb3d9496db4e841d557d7a196, entries=150, sequenceid=92, filesize=11.7 K 2024-11-18T20:13:07,140 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/.tmp/C/a48911d413104538b1b658433f9a7afe as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/C/a48911d413104538b1b658433f9a7afe 2024-11-18T20:13:07,147 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/C/a48911d413104538b1b658433f9a7afe, entries=150, sequenceid=92, filesize=11.7 K 2024-11-18T20:13:07,148 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=127.47 KB/130530 for 294b8f086325976977089df95da38d70 in 296ms, sequenceid=92, compaction requested=false 2024-11-18T20:13:07,148 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 294b8f086325976977089df95da38d70: 2024-11-18T20:13:07,243 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35963 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-18T20:13:07,243 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70. 2024-11-18T20:13:07,243 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2902): Flushing 294b8f086325976977089df95da38d70 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-18T20:13:07,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/.tmp/A/f0a1d5aafb0840f5a1fad13c65d79fbf is 50, key is test_row_0/A:col10/1731960786942/Put/seqid=0 2024-11-18T20:13:07,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(8855): Flush requested on 294b8f086325976977089df95da38d70 2024-11-18T20:13:07,278 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70. as already flushing 2024-11-18T20:13:07,292 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742083_1259 (size=12001) 2024-11-18T20:13:07,293 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=115 (bloomFilter=true), to=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/.tmp/A/f0a1d5aafb0840f5a1fad13c65d79fbf 2024-11-18T20:13:07,308 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:07,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54570 deadline: 1731960847297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:07,308 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:07,308 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:07,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54602 deadline: 1731960847297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:07,308 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:07,308 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:13:07,308 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:07,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54532 deadline: 1731960847299, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:07,309 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:07,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54600 deadline: 1731960847299, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:07,309 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:07,309 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:07,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54614 deadline: 1731960847304, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:07,309 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:07,309 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:13:07,309 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:07,309 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:07,309 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:13:07,310 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:07,310 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:07,310 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:13:07,313 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:07,313 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:07,313 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:13:07,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/.tmp/B/5f5c0adbe79d490bb9b0beef5b618308 is 50, key is test_row_0/B:col10/1731960786942/Put/seqid=0 2024-11-18T20:13:07,326 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742084_1260 (size=12001) 2024-11-18T20:13:07,415 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:07,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54600 deadline: 1731960847413, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:07,415 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:07,415 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:07,415 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:07,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54570 deadline: 1731960847414, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:07,415 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:13:07,416 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:07,416 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:07,416 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:07,416 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:13:07,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54532 deadline: 1731960847414, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:07,416 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:07,416 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:07,416 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:07,416 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:13:07,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54614 deadline: 1731960847414, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:07,416 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:07,416 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:07,416 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:13:07,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-18T20:13:07,426 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:07,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54602 deadline: 1731960847423, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:07,427 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:07,427 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:07,427 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:13:07,624 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:07,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54600 deadline: 1731960847622, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:07,624 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:07,624 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:07,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54570 deadline: 1731960847623, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:07,624 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:07,624 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:13:07,625 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:07,625 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:07,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54532 deadline: 1731960847623, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:07,625 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:07,625 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:13:07,625 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:07,625 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:07,625 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:13:07,625 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:07,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54614 deadline: 1731960847623, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:07,626 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:07,626 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:07,626 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:13:07,635 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:07,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54602 deadline: 1731960847633, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:07,635 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:07,635 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:07,635 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:13:07,728 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=115 (bloomFilter=true), to=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/.tmp/B/5f5c0adbe79d490bb9b0beef5b618308 2024-11-18T20:13:07,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/.tmp/C/b360175e4f6448bd8e5deb4e8388de99 is 50, key is test_row_0/C:col10/1731960786942/Put/seqid=0 2024-11-18T20:13:07,738 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742085_1261 (size=12001) 2024-11-18T20:13:07,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-18T20:13:07,936 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:07,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54600 deadline: 1731960847933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:07,936 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:07,936 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:07,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54570 deadline: 1731960847933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:07,936 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:07,937 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:13:07,937 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:07,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54532 deadline: 1731960847934, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:07,937 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:07,937 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:07,937 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:07,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54614 deadline: 1731960847934, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:07,937 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:13:07,937 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:07,937 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:07,937 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:07,937 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:13:07,937 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:07,937 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:13:07,944 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:07,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54602 deadline: 1731960847943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:07,944 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:07,944 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:07,944 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:13:08,140 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=115 (bloomFilter=true), to=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/.tmp/C/b360175e4f6448bd8e5deb4e8388de99 2024-11-18T20:13:08,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/.tmp/A/f0a1d5aafb0840f5a1fad13c65d79fbf as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/A/f0a1d5aafb0840f5a1fad13c65d79fbf 2024-11-18T20:13:08,150 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/A/f0a1d5aafb0840f5a1fad13c65d79fbf, entries=150, sequenceid=115, filesize=11.7 K 2024-11-18T20:13:08,150 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/.tmp/B/5f5c0adbe79d490bb9b0beef5b618308 as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/B/5f5c0adbe79d490bb9b0beef5b618308 2024-11-18T20:13:08,154 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/B/5f5c0adbe79d490bb9b0beef5b618308, entries=150, sequenceid=115, filesize=11.7 K 2024-11-18T20:13:08,155 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/.tmp/C/b360175e4f6448bd8e5deb4e8388de99 as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/C/b360175e4f6448bd8e5deb4e8388de99 2024-11-18T20:13:08,159 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/C/b360175e4f6448bd8e5deb4e8388de99, entries=150, sequenceid=115, filesize=11.7 K 2024-11-18T20:13:08,160 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=73.80 KB/75570 for 294b8f086325976977089df95da38d70 in 917ms, sequenceid=115, compaction requested=true 2024-11-18T20:13:08,160 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2603): Flush status journal for 294b8f086325976977089df95da38d70: 2024-11-18T20:13:08,160 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70. 2024-11-18T20:13:08,160 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=65 2024-11-18T20:13:08,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.HMaster(4169): Remote procedure done, pid=65 2024-11-18T20:13:08,163 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=65, resume processing ppid=64 2024-11-18T20:13:08,164 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=65, ppid=64, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3820 sec 2024-11-18T20:13:08,165 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=64, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees in 1.3900 sec 2024-11-18T20:13:08,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(8855): Flush requested on 294b8f086325976977089df95da38d70 2024-11-18T20:13:08,446 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-18T20:13:08,446 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 294b8f086325976977089df95da38d70 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-11-18T20:13:08,451 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/.tmp/A/3f0423b700404204bee394167d8140ce is 50, key is test_row_0/A:col10/1731960788446/Put/seqid=0 2024-11-18T20:13:08,455 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742086_1262 (size=14441) 2024-11-18T20:13:08,470 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:08,470 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:08,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54570 deadline: 1731960848465, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:08,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54532 deadline: 1731960848465, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:08,470 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:08,470 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:08,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54614 deadline: 1731960848466, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:08,470 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:08,470 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:13:08,470 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:08,470 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:08,470 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:13:08,470 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:08,470 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:08,470 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:13:08,473 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:08,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54600 deadline: 1731960848470, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:08,474 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:08,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54602 deadline: 1731960848470, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:08,474 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:08,474 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:08,474 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:13:08,474 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:08,474 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:08,474 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:13:08,576 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:08,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54570 deadline: 1731960848573, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:08,577 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:08,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54532 deadline: 1731960848573, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:08,577 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:08,577 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:08,577 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:08,577 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:13:08,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54614 deadline: 1731960848573, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:08,577 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:08,577 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:08,577 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:13:08,578 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:08,578 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:08,578 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:13:08,586 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:08,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54600 deadline: 1731960848583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:08,587 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:08,587 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:08,587 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:08,587 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:13:08,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54602 deadline: 1731960848583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:08,587 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:08,587 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:08,587 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:13:08,786 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:08,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54570 deadline: 1731960848783, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:08,786 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:08,786 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:08,786 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:08,786 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:13:08,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54532 deadline: 1731960848783, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:08,787 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:08,787 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:08,787 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:08,787 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:13:08,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54614 deadline: 1731960848783, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:08,787 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:08,787 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:08,787 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:13:08,796 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:08,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54600 deadline: 1731960848793, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:08,796 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:08,796 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:08,796 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:08,796 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:13:08,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54602 deadline: 1731960848793, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:08,797 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:08,797 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:08,797 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:13:08,855 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=132 (bloomFilter=true), to=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/.tmp/A/3f0423b700404204bee394167d8140ce 2024-11-18T20:13:08,864 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/.tmp/B/9f3850dd8b8f44c896af45ecfb34d569 is 50, key is test_row_0/B:col10/1731960788446/Put/seqid=0 2024-11-18T20:13:08,870 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742087_1263 (size=12101) 2024-11-18T20:13:08,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-18T20:13:08,943 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-18T20:13:08,943 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-18T20:13:08,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] procedure2.ProcedureExecutor(1139): Stored pid=66, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees 2024-11-18T20:13:08,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-18T20:13:08,945 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=66, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-18T20:13:08,946 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=66, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-18T20:13:08,946 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=67, ppid=66, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-18T20:13:09,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-18T20:13:09,097 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:09,097 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35963 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-18T20:13:09,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54532 deadline: 1731960849093, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:09,097 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:09,097 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70. 2024-11-18T20:13:09,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54570 deadline: 1731960849093, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:09,098 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70. as already flushing 2024-11-18T20:13:09,098 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:09,098 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70. 2024-11-18T20:13:09,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54614 deadline: 1731960849093, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:09,098 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 294b8f086325976977089df95da38d70, NAME => 'TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:13:09,098 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:09,098 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 294b8f086325976977089df95da38d70, NAME => 'TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:13:09,098 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:09,098 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:13:09,098 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:09,098 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:09,098 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:09,098 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:13:09,098 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:09,098 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:13:09,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 294b8f086325976977089df95da38d70, NAME => 'TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 294b8f086325976977089df95da38d70, NAME => 'TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:13:09,106 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:09,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54600 deadline: 1731960849103, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:09,106 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:09,106 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:09,106 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:09,106 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:13:09,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54602 deadline: 1731960849103, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:09,107 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:09,107 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:09,107 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:13:09,250 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35963 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-18T20:13:09,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70. 2024-11-18T20:13:09,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70. as already flushing 2024-11-18T20:13:09,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70. 2024-11-18T20:13:09,250 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 294b8f086325976977089df95da38d70, NAME => 'TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:13:09,251 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 294b8f086325976977089df95da38d70, NAME => 'TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:13:09,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 294b8f086325976977089df95da38d70, NAME => 'TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 294b8f086325976977089df95da38d70, NAME => 'TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:13:09,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-18T20:13:09,271 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=132 (bloomFilter=true), to=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/.tmp/B/9f3850dd8b8f44c896af45ecfb34d569 2024-11-18T20:13:09,278 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/.tmp/C/68e856d8a761480fbb963bb5f684ac45 is 50, key is test_row_0/C:col10/1731960788446/Put/seqid=0 2024-11-18T20:13:09,281 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742088_1264 (size=12101) 2024-11-18T20:13:09,282 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=132 (bloomFilter=true), to=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/.tmp/C/68e856d8a761480fbb963bb5f684ac45 2024-11-18T20:13:09,286 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/.tmp/A/3f0423b700404204bee394167d8140ce as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/A/3f0423b700404204bee394167d8140ce 2024-11-18T20:13:09,290 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/A/3f0423b700404204bee394167d8140ce, entries=200, sequenceid=132, filesize=14.1 K 2024-11-18T20:13:09,291 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/.tmp/B/9f3850dd8b8f44c896af45ecfb34d569 as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/B/9f3850dd8b8f44c896af45ecfb34d569 2024-11-18T20:13:09,294 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/B/9f3850dd8b8f44c896af45ecfb34d569, entries=150, sequenceid=132, filesize=11.8 K 2024-11-18T20:13:09,295 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/.tmp/C/68e856d8a761480fbb963bb5f684ac45 as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/C/68e856d8a761480fbb963bb5f684ac45 2024-11-18T20:13:09,299 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/C/68e856d8a761480fbb963bb5f684ac45, entries=150, sequenceid=132, filesize=11.8 K 2024-11-18T20:13:09,300 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~246.80 KB/252720, currentSize=114.05 KB/116790 for 294b8f086325976977089df95da38d70 in 854ms, sequenceid=132, compaction requested=true 2024-11-18T20:13:09,300 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 294b8f086325976977089df95da38d70: 2024-11-18T20:13:09,300 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 294b8f086325976977089df95da38d70:A, priority=-2147483648, current under compaction store size is 1 2024-11-18T20:13:09,300 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:13:09,300 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-18T20:13:09,300 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 294b8f086325976977089df95da38d70:B, priority=-2147483648, current under compaction store size is 2 2024-11-18T20:13:09,300 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:13:09,300 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-18T20:13:09,300 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 294b8f086325976977089df95da38d70:C, priority=-2147483648, current under compaction store size is 3 2024-11-18T20:13:09,300 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T20:13:09,301 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50582 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-18T20:13:09,301 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HStore(1541): 294b8f086325976977089df95da38d70/A is initiating minor compaction (all files) 2024-11-18T20:13:09,301 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 294b8f086325976977089df95da38d70/A in TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70. 2024-11-18T20:13:09,301 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/A/56e41c81598b46ac93b521ec008bc0fb, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/A/e4181afa412d49b98b17bbf3a2c56f2a, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/A/f0a1d5aafb0840f5a1fad13c65d79fbf, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/A/3f0423b700404204bee394167d8140ce] into tmpdir=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/.tmp, totalSize=49.4 K 2024-11-18T20:13:09,302 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48242 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-18T20:13:09,302 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HStore(1541): 294b8f086325976977089df95da38d70/B is initiating minor compaction (all files) 2024-11-18T20:13:09,302 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 294b8f086325976977089df95da38d70/B in TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70. 2024-11-18T20:13:09,302 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/B/87bc89a38c1148d2859fd397687f49ae, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/B/2099464fb3d9496db4e841d557d7a196, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/B/5f5c0adbe79d490bb9b0beef5b618308, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/B/9f3850dd8b8f44c896af45ecfb34d569] into tmpdir=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/.tmp, totalSize=47.1 K 2024-11-18T20:13:09,302 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] compactions.Compactor(225): Compacting 56e41c81598b46ac93b521ec008bc0fb, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=75, earliestPutTs=1731960785864 2024-11-18T20:13:09,302 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.Compactor(225): Compacting 87bc89a38c1148d2859fd397687f49ae, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=75, earliestPutTs=1731960785864 2024-11-18T20:13:09,302 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] compactions.Compactor(225): Compacting e4181afa412d49b98b17bbf3a2c56f2a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1731960786517 2024-11-18T20:13:09,302 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.Compactor(225): Compacting 2099464fb3d9496db4e841d557d7a196, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1731960786517 2024-11-18T20:13:09,302 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] compactions.Compactor(225): Compacting f0a1d5aafb0840f5a1fad13c65d79fbf, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=115, earliestPutTs=1731960786942 2024-11-18T20:13:09,303 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.Compactor(225): Compacting 5f5c0adbe79d490bb9b0beef5b618308, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=115, earliestPutTs=1731960786942 2024-11-18T20:13:09,303 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3f0423b700404204bee394167d8140ce, keycount=200, bloomtype=ROW, size=14.1 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1731960787296 2024-11-18T20:13:09,303 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.Compactor(225): Compacting 9f3850dd8b8f44c896af45ecfb34d569, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1731960787296 2024-11-18T20:13:09,312 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 294b8f086325976977089df95da38d70#B#compaction#221 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:13:09,313 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/.tmp/B/30488c808ef74e198607ddf904974164 is 50, key is test_row_0/B:col10/1731960788446/Put/seqid=0 2024-11-18T20:13:09,316 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 294b8f086325976977089df95da38d70#A#compaction#222 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:13:09,316 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/.tmp/A/278c8263546e4e03a8190fffe4c44dde is 50, key is test_row_0/A:col10/1731960788446/Put/seqid=0 2024-11-18T20:13:09,318 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742089_1265 (size=12375) 2024-11-18T20:13:09,322 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742090_1266 (size=12375) 2024-11-18T20:13:09,325 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/.tmp/B/30488c808ef74e198607ddf904974164 as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/B/30488c808ef74e198607ddf904974164 2024-11-18T20:13:09,327 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/.tmp/A/278c8263546e4e03a8190fffe4c44dde as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/A/278c8263546e4e03a8190fffe4c44dde 2024-11-18T20:13:09,330 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 294b8f086325976977089df95da38d70/B of 294b8f086325976977089df95da38d70 into 30488c808ef74e198607ddf904974164(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:13:09,330 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 294b8f086325976977089df95da38d70: 2024-11-18T20:13:09,330 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., storeName=294b8f086325976977089df95da38d70/B, priority=12, startTime=1731960789300; duration=0sec 2024-11-18T20:13:09,330 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T20:13:09,330 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 294b8f086325976977089df95da38d70:B 2024-11-18T20:13:09,330 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-18T20:13:09,332 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48242 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-18T20:13:09,332 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HStore(1541): 294b8f086325976977089df95da38d70/C is initiating minor compaction (all files) 2024-11-18T20:13:09,332 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 294b8f086325976977089df95da38d70/C in TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70. 2024-11-18T20:13:09,333 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/C/33d7ec8003794e2a9c97e3019a47912d, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/C/a48911d413104538b1b658433f9a7afe, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/C/b360175e4f6448bd8e5deb4e8388de99, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/C/68e856d8a761480fbb963bb5f684ac45] into tmpdir=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/.tmp, totalSize=47.1 K 2024-11-18T20:13:09,333 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.Compactor(225): Compacting 33d7ec8003794e2a9c97e3019a47912d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=75, earliestPutTs=1731960785864 2024-11-18T20:13:09,333 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.Compactor(225): Compacting a48911d413104538b1b658433f9a7afe, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1731960786517 2024-11-18T20:13:09,333 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.Compactor(225): Compacting b360175e4f6448bd8e5deb4e8388de99, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=115, earliestPutTs=1731960786942 2024-11-18T20:13:09,334 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.Compactor(225): Compacting 68e856d8a761480fbb963bb5f684ac45, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1731960787296 2024-11-18T20:13:09,334 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 294b8f086325976977089df95da38d70/A of 294b8f086325976977089df95da38d70 into 278c8263546e4e03a8190fffe4c44dde(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:13:09,334 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 294b8f086325976977089df95da38d70: 2024-11-18T20:13:09,334 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., storeName=294b8f086325976977089df95da38d70/A, priority=12, startTime=1731960789300; duration=0sec 2024-11-18T20:13:09,334 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:13:09,334 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 294b8f086325976977089df95da38d70:A 2024-11-18T20:13:09,342 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 294b8f086325976977089df95da38d70#C#compaction#223 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:13:09,342 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/.tmp/C/a6575a8c89cf4439bec4adfb0f820bb4 is 50, key is test_row_0/C:col10/1731960788446/Put/seqid=0 2024-11-18T20:13:09,347 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742091_1267 (size=12375) 2024-11-18T20:13:09,403 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35963 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-18T20:13:09,403 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70. 2024-11-18T20:13:09,403 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2902): Flushing 294b8f086325976977089df95da38d70 3/3 column families, dataSize=114.05 KB heapSize=299.58 KB 2024-11-18T20:13:09,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/.tmp/A/f62e78b53c944147aa95cb47c8f64e96 is 50, key is test_row_0/A:col10/1731960788464/Put/seqid=0 2024-11-18T20:13:09,440 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742092_1268 (size=12151) 2024-11-18T20:13:09,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-18T20:13:09,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(8855): Flush requested on 294b8f086325976977089df95da38d70 2024-11-18T20:13:09,605 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70. as already flushing 2024-11-18T20:13:09,624 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:09,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54532 deadline: 1731960849619, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:09,624 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:09,624 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:09,624 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:09,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54614 deadline: 1731960849619, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:09,624 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:13:09,625 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:09,625 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:09,625 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:09,625 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:13:09,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54570 deadline: 1731960849620, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:09,625 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:09,625 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:09,625 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:09,625 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:13:09,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54600 deadline: 1731960849621, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:09,625 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:09,625 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:09,625 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:13:09,626 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:09,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54602 deadline: 1731960849622, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:09,626 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:09,626 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:09,626 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:13:09,735 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:09,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54532 deadline: 1731960849733, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:09,736 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:09,736 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:09,736 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:09,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54614 deadline: 1731960849733, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:09,736 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:13:09,736 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:09,736 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:09,736 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:09,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54570 deadline: 1731960849733, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:09,736 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:13:09,736 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:09,736 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:09,736 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:09,736 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:13:09,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54600 deadline: 1731960849733, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:09,737 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:09,737 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:09,737 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:09,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54602 deadline: 1731960849733, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:09,737 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:13:09,737 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:09,737 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:09,737 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:13:09,752 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/.tmp/C/a6575a8c89cf4439bec4adfb0f820bb4 as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/C/a6575a8c89cf4439bec4adfb0f820bb4 2024-11-18T20:13:09,756 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 294b8f086325976977089df95da38d70/C of 294b8f086325976977089df95da38d70 into a6575a8c89cf4439bec4adfb0f820bb4(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:13:09,757 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 294b8f086325976977089df95da38d70: 2024-11-18T20:13:09,757 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., storeName=294b8f086325976977089df95da38d70/C, priority=12, startTime=1731960789300; duration=0sec 2024-11-18T20:13:09,757 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:13:09,757 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 294b8f086325976977089df95da38d70:C 2024-11-18T20:13:09,841 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=154 (bloomFilter=true), to=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/.tmp/A/f62e78b53c944147aa95cb47c8f64e96 2024-11-18T20:13:09,848 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/.tmp/B/65accc17d5364351a4b5440e366bcbc7 is 50, key is test_row_0/B:col10/1731960788464/Put/seqid=0 2024-11-18T20:13:09,854 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742093_1269 (size=12151) 2024-11-18T20:13:09,855 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=154 (bloomFilter=true), to=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/.tmp/B/65accc17d5364351a4b5440e366bcbc7 2024-11-18T20:13:09,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/.tmp/C/2e557bf37f304aa5a2c3070e9c42ded9 is 50, key is test_row_0/C:col10/1731960788464/Put/seqid=0 2024-11-18T20:13:09,865 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742094_1270 (size=12151) 2024-11-18T20:13:09,865 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=154 (bloomFilter=true), to=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/.tmp/C/2e557bf37f304aa5a2c3070e9c42ded9 2024-11-18T20:13:09,869 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/.tmp/A/f62e78b53c944147aa95cb47c8f64e96 as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/A/f62e78b53c944147aa95cb47c8f64e96 2024-11-18T20:13:09,874 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/A/f62e78b53c944147aa95cb47c8f64e96, entries=150, sequenceid=154, filesize=11.9 K 2024-11-18T20:13:09,876 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/.tmp/B/65accc17d5364351a4b5440e366bcbc7 as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/B/65accc17d5364351a4b5440e366bcbc7 2024-11-18T20:13:09,881 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/B/65accc17d5364351a4b5440e366bcbc7, entries=150, sequenceid=154, filesize=11.9 K 2024-11-18T20:13:09,882 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/.tmp/C/2e557bf37f304aa5a2c3070e9c42ded9 as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/C/2e557bf37f304aa5a2c3070e9c42ded9 2024-11-18T20:13:09,890 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/C/2e557bf37f304aa5a2c3070e9c42ded9, entries=150, sequenceid=154, filesize=11.9 K 2024-11-18T20:13:09,891 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~299.53 KB/306720, currentSize=93.93 KB/96180 for 294b8f086325976977089df95da38d70 in 488ms, sequenceid=154, compaction requested=false 2024-11-18T20:13:09,891 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2603): Flush status journal for 294b8f086325976977089df95da38d70: 2024-11-18T20:13:09,891 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70. 2024-11-18T20:13:09,891 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=67 2024-11-18T20:13:09,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.HMaster(4169): Remote procedure done, pid=67 2024-11-18T20:13:09,895 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=67, resume processing ppid=66 2024-11-18T20:13:09,895 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=67, ppid=66, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 947 msec 2024-11-18T20:13:09,897 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=66, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees in 953 msec 2024-11-18T20:13:09,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(8855): Flush requested on 294b8f086325976977089df95da38d70 2024-11-18T20:13:09,945 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-18T20:13:09,945 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 294b8f086325976977089df95da38d70 3/3 column families, dataSize=100.63 KB heapSize=264.42 KB 2024-11-18T20:13:09,950 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/.tmp/A/c40f358a3dac4deca6a5df11adbbef03 is 50, key is test_row_0/A:col10/1731960789944/Put/seqid=0 2024-11-18T20:13:09,955 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742095_1271 (size=12151) 2024-11-18T20:13:09,955 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=174 (bloomFilter=true), to=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/.tmp/A/c40f358a3dac4deca6a5df11adbbef03 2024-11-18T20:13:09,964 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/.tmp/B/3068410e267748008678ea24f73ddc40 is 50, key is test_row_0/B:col10/1731960789944/Put/seqid=0 2024-11-18T20:13:09,966 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:09,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54614 deadline: 1731960849961, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:09,966 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:09,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54602 deadline: 1731960849961, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:09,967 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:09,967 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:09,967 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:09,967 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:09,967 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:13:09,967 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:13:09,967 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742096_1272 (size=12151) 2024-11-18T20:13:09,967 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:09,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54570 deadline: 1731960849962, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:09,968 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:09,968 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:09,968 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:09,968 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:13:09,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54600 deadline: 1731960849964, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:09,968 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=174 (bloomFilter=true), to=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/.tmp/B/3068410e267748008678ea24f73ddc40 2024-11-18T20:13:09,968 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:09,968 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:09,968 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:13:09,969 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:09,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54532 deadline: 1731960849965, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:09,970 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:09,970 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:09,970 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:13:09,979 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/.tmp/C/028b665e5f3a4b3082348a8dd2ab2b6b is 50, key is test_row_0/C:col10/1731960789944/Put/seqid=0 2024-11-18T20:13:09,994 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742097_1273 (size=12151) 2024-11-18T20:13:10,075 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:10,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54602 deadline: 1731960850073, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:10,075 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:10,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54614 deadline: 1731960850073, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:10,075 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:10,075 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:10,075 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:13:10,075 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:10,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54570 deadline: 1731960850073, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:10,075 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:10,076 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:10,076 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:10,076 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:10,076 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:10,076 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:13:10,076 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:13:10,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54600 deadline: 1731960850073, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:10,076 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:10,076 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:10,076 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:10,076 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:13:10,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54532 deadline: 1731960850073, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:10,076 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:10,076 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:10,076 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:13:10,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-18T20:13:10,083 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-18T20:13:10,083 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-18T20:13:10,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] procedure2.ProcedureExecutor(1139): Stored pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees 2024-11-18T20:13:10,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-18T20:13:10,085 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-18T20:13:10,085 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-18T20:13:10,086 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=69, ppid=68, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-18T20:13:10,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-18T20:13:10,238 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35963 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-18T20:13:10,238 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70. 2024-11-18T20:13:10,238 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70. as already flushing 2024-11-18T20:13:10,238 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70. 2024-11-18T20:13:10,238 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 294b8f086325976977089df95da38d70, NAME => 'TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:13:10,238 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 294b8f086325976977089df95da38d70, NAME => 'TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:13:10,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 294b8f086325976977089df95da38d70, NAME => 'TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 294b8f086325976977089df95da38d70, NAME => 'TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:13:10,285 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:10,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54602 deadline: 1731960850283, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:10,286 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:10,286 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:10,286 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:13:10,286 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:10,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54614 deadline: 1731960850283, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:10,286 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:10,286 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:10,286 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:13:10,286 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:10,286 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:10,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54600 deadline: 1731960850284, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:10,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54570 deadline: 1731960850284, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:10,287 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:10,287 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:10,287 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:10,287 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:13:10,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54532 deadline: 1731960850284, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:10,287 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:10,287 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:10,287 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:13:10,287 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:10,287 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:10,287 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:13:10,391 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35963 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-18T20:13:10,391 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70. 2024-11-18T20:13:10,391 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70. as already flushing 2024-11-18T20:13:10,391 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70. 2024-11-18T20:13:10,391 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 294b8f086325976977089df95da38d70, NAME => 'TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:13:10,391 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 294b8f086325976977089df95da38d70, NAME => 'TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:13:10,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 294b8f086325976977089df95da38d70, NAME => 'TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 294b8f086325976977089df95da38d70, NAME => 'TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:13:10,395 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=174 (bloomFilter=true), to=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/.tmp/C/028b665e5f3a4b3082348a8dd2ab2b6b 2024-11-18T20:13:10,400 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/.tmp/A/c40f358a3dac4deca6a5df11adbbef03 as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/A/c40f358a3dac4deca6a5df11adbbef03 2024-11-18T20:13:10,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-18T20:13:10,404 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/A/c40f358a3dac4deca6a5df11adbbef03, entries=150, sequenceid=174, filesize=11.9 K 2024-11-18T20:13:10,405 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/.tmp/B/3068410e267748008678ea24f73ddc40 as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/B/3068410e267748008678ea24f73ddc40 2024-11-18T20:13:10,409 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/B/3068410e267748008678ea24f73ddc40, entries=150, sequenceid=174, filesize=11.9 K 2024-11-18T20:13:10,409 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/.tmp/C/028b665e5f3a4b3082348a8dd2ab2b6b as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/C/028b665e5f3a4b3082348a8dd2ab2b6b 2024-11-18T20:13:10,413 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/C/028b665e5f3a4b3082348a8dd2ab2b6b, entries=150, sequenceid=174, filesize=11.9 K 2024-11-18T20:13:10,414 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~281.95 KB/288720, currentSize=93.93 KB/96180 for 294b8f086325976977089df95da38d70 in 469ms, sequenceid=174, compaction requested=true 2024-11-18T20:13:10,414 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 294b8f086325976977089df95da38d70: 2024-11-18T20:13:10,414 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 294b8f086325976977089df95da38d70:A, priority=-2147483648, current under compaction store size is 1 2024-11-18T20:13:10,414 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T20:13:10,414 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:13:10,414 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 294b8f086325976977089df95da38d70:B, priority=-2147483648, current under compaction store size is 2 2024-11-18T20:13:10,414 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:13:10,415 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T20:13:10,415 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 294b8f086325976977089df95da38d70:C, priority=-2147483648, current under compaction store size is 3 2024-11-18T20:13:10,415 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T20:13:10,415 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36677 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T20:13:10,415 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36677 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T20:13:10,415 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HStore(1541): 294b8f086325976977089df95da38d70/A is initiating minor compaction (all files) 2024-11-18T20:13:10,415 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HStore(1541): 294b8f086325976977089df95da38d70/B is initiating minor compaction (all files) 2024-11-18T20:13:10,415 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 294b8f086325976977089df95da38d70/A in TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70. 2024-11-18T20:13:10,415 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 294b8f086325976977089df95da38d70/B in TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70. 2024-11-18T20:13:10,416 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/B/30488c808ef74e198607ddf904974164, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/B/65accc17d5364351a4b5440e366bcbc7, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/B/3068410e267748008678ea24f73ddc40] into tmpdir=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/.tmp, totalSize=35.8 K 2024-11-18T20:13:10,416 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/A/278c8263546e4e03a8190fffe4c44dde, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/A/f62e78b53c944147aa95cb47c8f64e96, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/A/c40f358a3dac4deca6a5df11adbbef03] into tmpdir=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/.tmp, totalSize=35.8 K 2024-11-18T20:13:10,416 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] compactions.Compactor(225): Compacting 278c8263546e4e03a8190fffe4c44dde, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1731960787296 2024-11-18T20:13:10,416 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.Compactor(225): Compacting 30488c808ef74e198607ddf904974164, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1731960787296 2024-11-18T20:13:10,416 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] compactions.Compactor(225): Compacting f62e78b53c944147aa95cb47c8f64e96, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1731960788464 2024-11-18T20:13:10,416 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.Compactor(225): Compacting 65accc17d5364351a4b5440e366bcbc7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1731960788464 2024-11-18T20:13:10,416 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] compactions.Compactor(225): Compacting c40f358a3dac4deca6a5df11adbbef03, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=174, earliestPutTs=1731960789619 2024-11-18T20:13:10,416 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.Compactor(225): Compacting 3068410e267748008678ea24f73ddc40, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=174, earliestPutTs=1731960789619 2024-11-18T20:13:10,424 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 294b8f086325976977089df95da38d70#A#compaction#230 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:13:10,424 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 294b8f086325976977089df95da38d70#B#compaction#231 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:13:10,424 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/.tmp/A/e53dd6b2568444f391f1c1726107c192 is 50, key is test_row_0/A:col10/1731960789944/Put/seqid=0 2024-11-18T20:13:10,425 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/.tmp/B/ecfb2b52d5d94baeb8684fdb29c9fa8c is 50, key is test_row_0/B:col10/1731960789944/Put/seqid=0 2024-11-18T20:13:10,429 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742099_1275 (size=12527) 2024-11-18T20:13:10,438 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742098_1274 (size=12527) 2024-11-18T20:13:10,544 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35963 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-18T20:13:10,544 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70. 2024-11-18T20:13:10,544 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2902): Flushing 294b8f086325976977089df95da38d70 3/3 column families, dataSize=93.93 KB heapSize=246.84 KB 2024-11-18T20:13:10,549 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/.tmp/A/57592715ce4b460db24a60d036c13459 is 50, key is test_row_0/A:col10/1731960789954/Put/seqid=0 2024-11-18T20:13:10,553 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742100_1276 (size=12151) 2024-11-18T20:13:10,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(8855): Flush requested on 294b8f086325976977089df95da38d70 2024-11-18T20:13:10,596 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70. as already flushing 2024-11-18T20:13:10,613 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:10,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54570 deadline: 1731960850608, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:10,613 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:10,613 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:10,613 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:13:10,616 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:10,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54614 deadline: 1731960850610, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:10,616 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:10,616 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:10,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54600 deadline: 1731960850610, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:10,616 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:10,616 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:13:10,616 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:10,616 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:10,616 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:10,616 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:13:10,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54532 deadline: 1731960850611, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:10,616 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:10,616 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:10,616 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:10,616 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:13:10,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54602 deadline: 1731960850612, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:10,617 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:10,617 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:10,617 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:13:10,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-18T20:13:10,725 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:10,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54570 deadline: 1731960850723, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:10,725 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:10,725 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:10,726 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:10,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54614 deadline: 1731960850723, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:10,726 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:13:10,726 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:10,726 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:10,726 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:10,726 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:13:10,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54600 deadline: 1731960850723, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:10,726 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:10,726 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:10,726 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:10,726 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:13:10,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54532 deadline: 1731960850723, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:10,726 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:10,726 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:10,726 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:10,726 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:13:10,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54602 deadline: 1731960850723, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:10,727 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:10,727 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:10,727 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:13:10,834 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/.tmp/B/ecfb2b52d5d94baeb8684fdb29c9fa8c as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/B/ecfb2b52d5d94baeb8684fdb29c9fa8c 2024-11-18T20:13:10,838 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 294b8f086325976977089df95da38d70/B of 294b8f086325976977089df95da38d70 into ecfb2b52d5d94baeb8684fdb29c9fa8c(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:13:10,838 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 294b8f086325976977089df95da38d70: 2024-11-18T20:13:10,838 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., storeName=294b8f086325976977089df95da38d70/B, priority=13, startTime=1731960790414; duration=0sec 2024-11-18T20:13:10,838 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T20:13:10,838 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 294b8f086325976977089df95da38d70:B 2024-11-18T20:13:10,838 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T20:13:10,839 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36677 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T20:13:10,840 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HStore(1541): 294b8f086325976977089df95da38d70/C is initiating minor compaction (all files) 2024-11-18T20:13:10,840 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 294b8f086325976977089df95da38d70/C in TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70. 2024-11-18T20:13:10,840 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/C/a6575a8c89cf4439bec4adfb0f820bb4, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/C/2e557bf37f304aa5a2c3070e9c42ded9, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/C/028b665e5f3a4b3082348a8dd2ab2b6b] into tmpdir=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/.tmp, totalSize=35.8 K 2024-11-18T20:13:10,840 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.Compactor(225): Compacting a6575a8c89cf4439bec4adfb0f820bb4, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1731960787296 2024-11-18T20:13:10,840 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.Compactor(225): Compacting 2e557bf37f304aa5a2c3070e9c42ded9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1731960788464 2024-11-18T20:13:10,841 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.Compactor(225): Compacting 028b665e5f3a4b3082348a8dd2ab2b6b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=174, earliestPutTs=1731960789619 2024-11-18T20:13:10,843 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/.tmp/A/e53dd6b2568444f391f1c1726107c192 as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/A/e53dd6b2568444f391f1c1726107c192 2024-11-18T20:13:10,847 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 294b8f086325976977089df95da38d70/A of 294b8f086325976977089df95da38d70 into e53dd6b2568444f391f1c1726107c192(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:13:10,847 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 294b8f086325976977089df95da38d70: 2024-11-18T20:13:10,847 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., storeName=294b8f086325976977089df95da38d70/A, priority=13, startTime=1731960790414; duration=0sec 2024-11-18T20:13:10,847 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:13:10,847 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 294b8f086325976977089df95da38d70:A 2024-11-18T20:13:10,848 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 294b8f086325976977089df95da38d70#C#compaction#233 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:13:10,848 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/.tmp/C/bfbd2f8a966b4b348cc16e23b393b87d is 50, key is test_row_0/C:col10/1731960789944/Put/seqid=0 2024-11-18T20:13:10,851 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742101_1277 (size=12527) 2024-11-18T20:13:10,936 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:10,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54614 deadline: 1731960850933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:10,936 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:10,936 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:10,936 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:13:10,936 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:10,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54570 deadline: 1731960850933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:10,937 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:10,937 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:10,937 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:13:10,937 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:10,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54600 deadline: 1731960850933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:10,937 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:10,937 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:10,937 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:13:10,937 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:10,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54602 deadline: 1731960850933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:10,938 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:10,938 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:10,938 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:13:10,938 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:10,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54532 deadline: 1731960850933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:10,938 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:10,938 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:10,938 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:13:10,954 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=191 (bloomFilter=true), to=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/.tmp/A/57592715ce4b460db24a60d036c13459 2024-11-18T20:13:10,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/.tmp/B/c79f73571fb04b528088d8034c5153e1 is 50, key is test_row_0/B:col10/1731960789954/Put/seqid=0 2024-11-18T20:13:10,965 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742102_1278 (size=12151) 2024-11-18T20:13:11,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-18T20:13:11,244 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:11,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54570 deadline: 1731960851243, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:11,245 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:11,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54614 deadline: 1731960851243, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:11,245 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:11,245 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:11,245 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:13:11,245 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:11,245 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:11,245 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:11,245 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:13:11,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54600 deadline: 1731960851243, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:11,245 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:11,245 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:11,245 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:11,245 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:13:11,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54602 deadline: 1731960851243, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:11,246 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:11,246 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:11,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54532 deadline: 1731960851243, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:11,246 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:11,246 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:13:11,246 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:11,246 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:11,246 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:13:11,256 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/.tmp/C/bfbd2f8a966b4b348cc16e23b393b87d as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/C/bfbd2f8a966b4b348cc16e23b393b87d 2024-11-18T20:13:11,261 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 294b8f086325976977089df95da38d70/C of 294b8f086325976977089df95da38d70 into bfbd2f8a966b4b348cc16e23b393b87d(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:13:11,261 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 294b8f086325976977089df95da38d70: 2024-11-18T20:13:11,261 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., storeName=294b8f086325976977089df95da38d70/C, priority=13, startTime=1731960790415; duration=0sec 2024-11-18T20:13:11,261 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:13:11,261 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 294b8f086325976977089df95da38d70:C 2024-11-18T20:13:11,365 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=191 (bloomFilter=true), to=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/.tmp/B/c79f73571fb04b528088d8034c5153e1 2024-11-18T20:13:11,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/.tmp/C/33b932f406b245c18ee7796a32cfbb45 is 50, key is test_row_0/C:col10/1731960789954/Put/seqid=0 2024-11-18T20:13:11,376 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742103_1279 (size=12151) 2024-11-18T20:13:11,755 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:11,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54570 deadline: 1731960851753, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:11,755 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:11,755 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:11,755 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:11,755 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:13:11,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54614 deadline: 1731960851753, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:11,756 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:11,756 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:11,756 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:13:11,756 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:11,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54602 deadline: 1731960851753, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:11,756 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:11,756 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:11,756 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:13:11,756 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:11,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54600 deadline: 1731960851753, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:11,756 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:11,756 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:11,756 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:11,756 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:13:11,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54532 deadline: 1731960851753, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:11,757 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:11,757 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:11,757 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:13:11,777 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=191 (bloomFilter=true), to=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/.tmp/C/33b932f406b245c18ee7796a32cfbb45 2024-11-18T20:13:11,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/.tmp/A/57592715ce4b460db24a60d036c13459 as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/A/57592715ce4b460db24a60d036c13459 2024-11-18T20:13:11,787 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/A/57592715ce4b460db24a60d036c13459, entries=150, sequenceid=191, filesize=11.9 K 2024-11-18T20:13:11,788 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/.tmp/B/c79f73571fb04b528088d8034c5153e1 as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/B/c79f73571fb04b528088d8034c5153e1 2024-11-18T20:13:11,792 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/B/c79f73571fb04b528088d8034c5153e1, entries=150, sequenceid=191, filesize=11.9 K 2024-11-18T20:13:11,793 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/.tmp/C/33b932f406b245c18ee7796a32cfbb45 as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/C/33b932f406b245c18ee7796a32cfbb45 2024-11-18T20:13:11,798 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/C/33b932f406b245c18ee7796a32cfbb45, entries=150, sequenceid=191, filesize=11.9 K 2024-11-18T20:13:11,799 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~246.80 KB/252720, currentSize=107.34 KB/109920 for 294b8f086325976977089df95da38d70 in 1255ms, sequenceid=191, compaction requested=false 2024-11-18T20:13:11,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2603): Flush status journal for 294b8f086325976977089df95da38d70: 2024-11-18T20:13:11,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70. 2024-11-18T20:13:11,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=69 2024-11-18T20:13:11,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.HMaster(4169): Remote procedure done, pid=69 2024-11-18T20:13:11,803 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=69, resume processing ppid=68 2024-11-18T20:13:11,803 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=69, ppid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7150 sec 2024-11-18T20:13:11,805 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees in 1.7210 sec 2024-11-18T20:13:12,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-18T20:13:12,234 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-18T20:13:12,234 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-18T20:13:12,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] procedure2.ProcedureExecutor(1139): Stored pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees 2024-11-18T20:13:12,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-18T20:13:12,236 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-18T20:13:12,236 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-18T20:13:12,237 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=71, ppid=70, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-18T20:13:12,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-18T20:13:12,389 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35963 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-18T20:13:12,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70. 2024-11-18T20:13:12,389 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2902): Flushing 294b8f086325976977089df95da38d70 3/3 column families, dataSize=107.34 KB heapSize=282 KB 2024-11-18T20:13:12,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/.tmp/A/2ca7ca5abfab41c49390187908b97e7b is 50, key is test_row_0/A:col10/1731960790609/Put/seqid=0 2024-11-18T20:13:12,410 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742104_1280 (size=12151) 2024-11-18T20:13:12,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-18T20:13:12,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(8855): Flush requested on 294b8f086325976977089df95da38d70 2024-11-18T20:13:12,766 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70. as already flushing 2024-11-18T20:13:12,811 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=213 (bloomFilter=true), to=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/.tmp/A/2ca7ca5abfab41c49390187908b97e7b 2024-11-18T20:13:12,814 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:12,814 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:12,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54602 deadline: 1731960852808, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:12,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54532 deadline: 1731960852809, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:12,815 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:12,815 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:12,815 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:12,815 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:12,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54600 deadline: 1731960852809, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:12,815 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:12,815 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:13:12,815 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:13:12,815 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:12,815 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:12,815 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:13:12,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/.tmp/B/2f9e132a392a4c159750eeda0a04a4e1 is 50, key is test_row_0/B:col10/1731960790609/Put/seqid=0 2024-11-18T20:13:12,820 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:12,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54570 deadline: 1731960852815, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:12,820 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:12,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54614 deadline: 1731960852815, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:12,821 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:12,821 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:12,821 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:13:12,821 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:12,821 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:12,821 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:13:12,824 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742105_1281 (size=12151) 2024-11-18T20:13:12,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-18T20:13:12,927 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:12,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54602 deadline: 1731960852923, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:12,927 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:12,927 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:12,927 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:13:12,927 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:12,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54532 deadline: 1731960852923, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:12,928 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:12,928 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:12,928 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:13:12,928 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:12,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54600 deadline: 1731960852923, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:12,928 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:12,928 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:12,928 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:13:12,928 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:12,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54570 deadline: 1731960852923, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:12,928 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:12,928 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:12,929 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:13:12,929 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:12,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54614 deadline: 1731960852923, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:12,929 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:12,929 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:12,929 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:13:13,135 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:13,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54602 deadline: 1731960853133, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:13,135 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:13,135 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:13,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54532 deadline: 1731960853133, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:13,135 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:13,135 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:13:13,136 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:13,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54570 deadline: 1731960853133, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:13,136 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:13,136 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:13,136 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:13:13,136 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:13,136 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:13,136 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:13,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54600 deadline: 1731960853134, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:13,136 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:13:13,136 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:13,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54614 deadline: 1731960853134, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:13,136 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:13,136 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:13,136 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:13,136 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:13:13,136 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:13,136 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:13:13,224 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=213 (bloomFilter=true), to=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/.tmp/B/2f9e132a392a4c159750eeda0a04a4e1 2024-11-18T20:13:13,231 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/.tmp/C/5de0ea295e084184b934a2060fd62e1e is 50, key is test_row_0/C:col10/1731960790609/Put/seqid=0 2024-11-18T20:13:13,238 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742106_1282 (size=12151) 2024-11-18T20:13:13,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-18T20:13:13,445 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:13,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54602 deadline: 1731960853443, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:13,446 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:13,446 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:13,446 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:13,446 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:13:13,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54532 deadline: 1731960853443, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:13,446 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:13,446 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:13,446 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:13,446 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:13:13,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54570 deadline: 1731960853444, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:13,446 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:13,446 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:13,446 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:13,446 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:13:13,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54600 deadline: 1731960853444, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:13,447 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:13,447 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:13,447 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:13,447 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:13:13,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54614 deadline: 1731960853444, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:13,447 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:13,447 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:13,447 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:13:13,640 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=213 (bloomFilter=true), to=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/.tmp/C/5de0ea295e084184b934a2060fd62e1e 2024-11-18T20:13:13,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/.tmp/A/2ca7ca5abfab41c49390187908b97e7b as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/A/2ca7ca5abfab41c49390187908b97e7b 2024-11-18T20:13:13,651 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/A/2ca7ca5abfab41c49390187908b97e7b, entries=150, sequenceid=213, filesize=11.9 K 2024-11-18T20:13:13,652 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/.tmp/B/2f9e132a392a4c159750eeda0a04a4e1 as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/B/2f9e132a392a4c159750eeda0a04a4e1 2024-11-18T20:13:13,655 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/B/2f9e132a392a4c159750eeda0a04a4e1, entries=150, sequenceid=213, filesize=11.9 K 2024-11-18T20:13:13,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/.tmp/C/5de0ea295e084184b934a2060fd62e1e as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/C/5de0ea295e084184b934a2060fd62e1e 2024-11-18T20:13:13,659 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/C/5de0ea295e084184b934a2060fd62e1e, entries=150, sequenceid=213, filesize=11.9 K 2024-11-18T20:13:13,660 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~281.95 KB/288720, currentSize=93.93 KB/96180 for 294b8f086325976977089df95da38d70 in 1271ms, sequenceid=213, compaction requested=true 2024-11-18T20:13:13,660 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2603): Flush status journal for 294b8f086325976977089df95da38d70: 2024-11-18T20:13:13,660 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70. 2024-11-18T20:13:13,660 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=71 2024-11-18T20:13:13,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.HMaster(4169): Remote procedure done, pid=71 2024-11-18T20:13:13,663 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=71, resume processing ppid=70 2024-11-18T20:13:13,663 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=71, ppid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4250 sec 2024-11-18T20:13:13,664 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees in 1.4290 sec 2024-11-18T20:13:13,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(8855): Flush requested on 294b8f086325976977089df95da38d70 2024-11-18T20:13:13,955 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-18T20:13:13,956 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 294b8f086325976977089df95da38d70 3/3 column families, dataSize=100.63 KB heapSize=264.42 KB 2024-11-18T20:13:13,961 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/.tmp/A/1a2f88f8f9b94234b63337e5b7a1da79 is 50, key is test_row_0/A:col10/1731960793956/Put/seqid=0 2024-11-18T20:13:13,964 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742107_1283 (size=16931) 2024-11-18T20:13:13,974 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:13,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54600 deadline: 1731960853969, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:13,974 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:13,974 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:13,974 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:13,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54570 deadline: 1731960853970, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:13,975 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:13:13,975 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:13,975 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:13,975 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:13,975 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:13:13,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54532 deadline: 1731960853970, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:13,975 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:13,975 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:13,975 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:13:13,975 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:13,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54602 deadline: 1731960853971, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:13,975 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:13,975 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:13,975 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:13:13,975 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:13,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54614 deadline: 1731960853971, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:13,976 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:13,976 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:13,976 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:13:14,085 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:14,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54600 deadline: 1731960854083, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:14,085 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:14,085 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:14,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54570 deadline: 1731960854083, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:14,085 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:14,085 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:13:14,085 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:14,085 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:14,085 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:14,085 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:13:14,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54532 deadline: 1731960854083, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:14,085 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:14,085 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:14,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54602 deadline: 1731960854083, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:14,085 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:14,086 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:13:14,086 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:14,086 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:14,086 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:14,086 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:13:14,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54614 deadline: 1731960854083, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:14,086 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:14,086 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:14,086 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:13:14,293 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:14,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54600 deadline: 1731960854293, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:14,294 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:14,294 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:14,294 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:13:14,296 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:14,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54570 deadline: 1731960854293, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:14,296 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:14,296 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:14,296 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:14,296 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:14,296 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:14,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54602 deadline: 1731960854293, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:14,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54614 deadline: 1731960854293, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:14,296 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:13:14,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54532 deadline: 1731960854293, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:14,296 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:14,296 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:14,297 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:14,297 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:14,297 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:14,297 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:13:14,297 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:13:14,297 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:14,297 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:13:14,365 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=233 (bloomFilter=true), to=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/.tmp/A/1a2f88f8f9b94234b63337e5b7a1da79 2024-11-18T20:13:14,371 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/.tmp/B/b57ed696696f47e5abb3f9cc7cb11b30 is 50, key is test_row_0/B:col10/1731960793956/Put/seqid=0 2024-11-18T20:13:14,374 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742108_1284 (size=12151) 2024-11-18T20:13:14,374 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=233 (bloomFilter=true), to=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/.tmp/B/b57ed696696f47e5abb3f9cc7cb11b30 2024-11-18T20:13:14,380 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/.tmp/C/47ea7d61bbb04e9ea44481e26914845a is 50, key is test_row_0/C:col10/1731960793956/Put/seqid=0 2024-11-18T20:13:14,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-18T20:13:14,384 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-18T20:13:14,384 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742109_1285 (size=12151) 2024-11-18T20:13:14,384 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-18T20:13:14,384 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=233 (bloomFilter=true), to=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/.tmp/C/47ea7d61bbb04e9ea44481e26914845a 2024-11-18T20:13:14,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] procedure2.ProcedureExecutor(1139): Stored pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees 2024-11-18T20:13:14,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-18T20:13:14,386 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-18T20:13:14,386 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-18T20:13:14,387 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=73, ppid=72, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-18T20:13:14,390 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/.tmp/A/1a2f88f8f9b94234b63337e5b7a1da79 as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/A/1a2f88f8f9b94234b63337e5b7a1da79 2024-11-18T20:13:14,395 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/A/1a2f88f8f9b94234b63337e5b7a1da79, entries=250, sequenceid=233, filesize=16.5 K 2024-11-18T20:13:14,395 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/.tmp/B/b57ed696696f47e5abb3f9cc7cb11b30 as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/B/b57ed696696f47e5abb3f9cc7cb11b30 2024-11-18T20:13:14,400 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/B/b57ed696696f47e5abb3f9cc7cb11b30, entries=150, sequenceid=233, filesize=11.9 K 2024-11-18T20:13:14,401 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/.tmp/C/47ea7d61bbb04e9ea44481e26914845a as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/C/47ea7d61bbb04e9ea44481e26914845a 2024-11-18T20:13:14,407 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/C/47ea7d61bbb04e9ea44481e26914845a, entries=150, sequenceid=233, filesize=11.9 K 2024-11-18T20:13:14,408 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~299.53 KB/306720, currentSize=87.22 KB/89310 for 294b8f086325976977089df95da38d70 in 453ms, sequenceid=233, compaction requested=true 2024-11-18T20:13:14,408 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 294b8f086325976977089df95da38d70: 2024-11-18T20:13:14,408 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 294b8f086325976977089df95da38d70:A, priority=-2147483648, current under compaction store size is 1 2024-11-18T20:13:14,408 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:13:14,408 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-18T20:13:14,408 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 294b8f086325976977089df95da38d70:B, priority=-2147483648, current under compaction store size is 2 2024-11-18T20:13:14,408 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:13:14,408 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-18T20:13:14,408 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 294b8f086325976977089df95da38d70:C, priority=-2147483648, current under compaction store size is 3 2024-11-18T20:13:14,408 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T20:13:14,409 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48980 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-18T20:13:14,409 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 53760 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-18T20:13:14,409 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HStore(1541): 294b8f086325976977089df95da38d70/B is initiating minor compaction (all files) 2024-11-18T20:13:14,409 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HStore(1541): 294b8f086325976977089df95da38d70/A is initiating minor compaction (all files) 2024-11-18T20:13:14,410 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 294b8f086325976977089df95da38d70/B in TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70. 2024-11-18T20:13:14,410 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 294b8f086325976977089df95da38d70/A in TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70. 2024-11-18T20:13:14,410 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/A/e53dd6b2568444f391f1c1726107c192, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/A/57592715ce4b460db24a60d036c13459, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/A/2ca7ca5abfab41c49390187908b97e7b, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/A/1a2f88f8f9b94234b63337e5b7a1da79] into tmpdir=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/.tmp, totalSize=52.5 K 2024-11-18T20:13:14,410 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/B/ecfb2b52d5d94baeb8684fdb29c9fa8c, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/B/c79f73571fb04b528088d8034c5153e1, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/B/2f9e132a392a4c159750eeda0a04a4e1, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/B/b57ed696696f47e5abb3f9cc7cb11b30] into tmpdir=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/.tmp, totalSize=47.8 K 2024-11-18T20:13:14,410 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.Compactor(225): Compacting ecfb2b52d5d94baeb8684fdb29c9fa8c, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=174, earliestPutTs=1731960789619 2024-11-18T20:13:14,410 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] compactions.Compactor(225): Compacting e53dd6b2568444f391f1c1726107c192, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=174, earliestPutTs=1731960789619 2024-11-18T20:13:14,411 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.Compactor(225): Compacting c79f73571fb04b528088d8034c5153e1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=191, earliestPutTs=1731960789954 2024-11-18T20:13:14,411 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] compactions.Compactor(225): Compacting 57592715ce4b460db24a60d036c13459, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=191, earliestPutTs=1731960789954 2024-11-18T20:13:14,412 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2ca7ca5abfab41c49390187908b97e7b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=213, earliestPutTs=1731960790609 2024-11-18T20:13:14,412 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.Compactor(225): Compacting 2f9e132a392a4c159750eeda0a04a4e1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=213, earliestPutTs=1731960790609 2024-11-18T20:13:14,412 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1a2f88f8f9b94234b63337e5b7a1da79, keycount=250, bloomtype=ROW, size=16.5 K, encoding=NONE, compression=NONE, seqNum=233, earliestPutTs=1731960792808 2024-11-18T20:13:14,412 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.Compactor(225): Compacting b57ed696696f47e5abb3f9cc7cb11b30, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=233, earliestPutTs=1731960792808 2024-11-18T20:13:14,423 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 294b8f086325976977089df95da38d70#A#compaction#242 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:13:14,424 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/.tmp/A/f336b89ef6f440c5ae78966493d3e3ec is 50, key is test_row_0/A:col10/1731960793956/Put/seqid=0 2024-11-18T20:13:14,431 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 294b8f086325976977089df95da38d70#B#compaction#243 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:13:14,431 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/.tmp/B/20466a25c4c74ded8a242b3da0aaa76a is 50, key is test_row_0/B:col10/1731960793956/Put/seqid=0 2024-11-18T20:13:14,432 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742110_1286 (size=12663) 2024-11-18T20:13:14,438 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742111_1287 (size=12663) 2024-11-18T20:13:14,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-18T20:13:14,538 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35963 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-18T20:13:14,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70. 2024-11-18T20:13:14,539 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2902): Flushing 294b8f086325976977089df95da38d70 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2024-11-18T20:13:14,544 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/.tmp/A/6bb29e1da64e40c0845d3a90926ee4ca is 50, key is test_row_0/A:col10/1731960793969/Put/seqid=0 2024-11-18T20:13:14,547 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742112_1288 (size=12151) 2024-11-18T20:13:14,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(8855): Flush requested on 294b8f086325976977089df95da38d70 2024-11-18T20:13:14,606 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70. as already flushing 2024-11-18T20:13:14,654 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:14,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54600 deadline: 1731960854650, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:14,654 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:14,654 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:14,654 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:14,655 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:13:14,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54602 deadline: 1731960854650, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:14,655 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:14,655 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:14,655 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:14,655 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:13:14,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54532 deadline: 1731960854650, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:14,655 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:14,655 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:14,655 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:14,655 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:13:14,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54570 deadline: 1731960854651, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:14,655 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:14,655 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:14,655 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:14,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54614 deadline: 1731960854651, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:14,655 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:13:14,656 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:14,656 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:14,656 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:13:14,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-18T20:13:14,766 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:14,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54600 deadline: 1731960854763, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:14,766 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:14,766 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:14,766 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:14,766 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:13:14,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54532 deadline: 1731960854763, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:14,766 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:14,766 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:14,766 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:14,766 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:13:14,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54602 deadline: 1731960854763, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:14,766 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:14,766 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:14,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54570 deadline: 1731960854763, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:14,767 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:14,767 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:13:14,767 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:14,767 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:14,767 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:14,767 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:13:14,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54614 deadline: 1731960854763, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:14,767 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:14,767 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:14,767 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:13:14,838 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/.tmp/A/f336b89ef6f440c5ae78966493d3e3ec as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/A/f336b89ef6f440c5ae78966493d3e3ec 2024-11-18T20:13:14,842 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/.tmp/B/20466a25c4c74ded8a242b3da0aaa76a as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/B/20466a25c4c74ded8a242b3da0aaa76a 2024-11-18T20:13:14,842 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 294b8f086325976977089df95da38d70/A of 294b8f086325976977089df95da38d70 into f336b89ef6f440c5ae78966493d3e3ec(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:13:14,842 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 294b8f086325976977089df95da38d70: 2024-11-18T20:13:14,842 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., storeName=294b8f086325976977089df95da38d70/A, priority=12, startTime=1731960794408; duration=0sec 2024-11-18T20:13:14,843 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T20:13:14,843 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 294b8f086325976977089df95da38d70:A 2024-11-18T20:13:14,843 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-18T20:13:14,849 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48980 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-18T20:13:14,849 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HStore(1541): 294b8f086325976977089df95da38d70/C is initiating minor compaction (all files) 2024-11-18T20:13:14,849 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 294b8f086325976977089df95da38d70/C in TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70. 2024-11-18T20:13:14,850 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/C/bfbd2f8a966b4b348cc16e23b393b87d, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/C/33b932f406b245c18ee7796a32cfbb45, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/C/5de0ea295e084184b934a2060fd62e1e, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/C/47ea7d61bbb04e9ea44481e26914845a] into tmpdir=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/.tmp, totalSize=47.8 K 2024-11-18T20:13:14,850 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] compactions.Compactor(225): Compacting bfbd2f8a966b4b348cc16e23b393b87d, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=174, earliestPutTs=1731960789619 2024-11-18T20:13:14,850 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] compactions.Compactor(225): Compacting 33b932f406b245c18ee7796a32cfbb45, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=191, earliestPutTs=1731960789954 2024-11-18T20:13:14,851 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5de0ea295e084184b934a2060fd62e1e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=213, earliestPutTs=1731960790609 2024-11-18T20:13:14,851 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] compactions.Compactor(225): Compacting 47ea7d61bbb04e9ea44481e26914845a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=233, earliestPutTs=1731960792808 2024-11-18T20:13:14,854 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 294b8f086325976977089df95da38d70/B of 294b8f086325976977089df95da38d70 into 20466a25c4c74ded8a242b3da0aaa76a(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:13:14,854 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 294b8f086325976977089df95da38d70: 2024-11-18T20:13:14,854 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., storeName=294b8f086325976977089df95da38d70/B, priority=12, startTime=1731960794408; duration=0sec 2024-11-18T20:13:14,854 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:13:14,854 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 294b8f086325976977089df95da38d70:B 2024-11-18T20:13:14,863 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 294b8f086325976977089df95da38d70#C#compaction#245 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:13:14,863 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/.tmp/C/61247116172143a5a44b686ab32303af is 50, key is test_row_0/C:col10/1731960793956/Put/seqid=0 2024-11-18T20:13:14,867 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742113_1289 (size=12663) 2024-11-18T20:13:14,949 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=249 (bloomFilter=true), to=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/.tmp/A/6bb29e1da64e40c0845d3a90926ee4ca 2024-11-18T20:13:14,955 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/.tmp/B/aa4a5d754cfc4a819020fe51384a849f is 50, key is test_row_0/B:col10/1731960793969/Put/seqid=0 2024-11-18T20:13:14,958 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742114_1290 (size=12151) 2024-11-18T20:13:14,975 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:14,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54532 deadline: 1731960854973, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:14,976 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:14,976 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:14,976 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:14,976 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:13:14,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54600 deadline: 1731960854973, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:14,976 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:14,976 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:14,976 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:14,976 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:13:14,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54602 deadline: 1731960854973, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:14,977 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:14,977 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:14,977 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:13:14,977 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:14,977 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:14,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54570 deadline: 1731960854973, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:14,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54614 deadline: 1731960854973, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:14,977 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:14,977 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:14,977 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:14,977 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:14,977 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:13:14,977 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:13:15,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-18T20:13:15,272 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/.tmp/C/61247116172143a5a44b686ab32303af as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/C/61247116172143a5a44b686ab32303af 2024-11-18T20:13:15,276 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 294b8f086325976977089df95da38d70/C of 294b8f086325976977089df95da38d70 into 61247116172143a5a44b686ab32303af(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:13:15,276 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 294b8f086325976977089df95da38d70: 2024-11-18T20:13:15,276 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., storeName=294b8f086325976977089df95da38d70/C, priority=12, startTime=1731960794408; duration=0sec 2024-11-18T20:13:15,276 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:13:15,276 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 294b8f086325976977089df95da38d70:C 2024-11-18T20:13:15,284 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:15,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54532 deadline: 1731960855283, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:15,284 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:15,284 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:15,284 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:13:15,286 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:15,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54600 deadline: 1731960855283, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:15,286 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:15,286 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:15,286 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:13:15,286 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:15,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54602 deadline: 1731960855283, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:15,287 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:15,287 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:15,287 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:15,287 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:13:15,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54614 deadline: 1731960855283, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:15,287 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:15,287 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:15,287 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:15,287 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:13:15,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54570 deadline: 1731960855283, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:15,287 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:15,287 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:15,287 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:13:15,359 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=249 (bloomFilter=true), to=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/.tmp/B/aa4a5d754cfc4a819020fe51384a849f 2024-11-18T20:13:15,366 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/.tmp/C/72b31fc27c0b4a30ab629298c92b8c50 is 50, key is test_row_0/C:col10/1731960793969/Put/seqid=0 2024-11-18T20:13:15,369 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742115_1291 (size=12151) 2024-11-18T20:13:15,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-18T20:13:15,770 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=249 (bloomFilter=true), to=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/.tmp/C/72b31fc27c0b4a30ab629298c92b8c50 2024-11-18T20:13:15,774 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/.tmp/A/6bb29e1da64e40c0845d3a90926ee4ca as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/A/6bb29e1da64e40c0845d3a90926ee4ca 2024-11-18T20:13:15,777 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/A/6bb29e1da64e40c0845d3a90926ee4ca, entries=150, sequenceid=249, filesize=11.9 K 2024-11-18T20:13:15,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/.tmp/B/aa4a5d754cfc4a819020fe51384a849f as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/B/aa4a5d754cfc4a819020fe51384a849f 2024-11-18T20:13:15,781 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/B/aa4a5d754cfc4a819020fe51384a849f, entries=150, sequenceid=249, filesize=11.9 K 2024-11-18T20:13:15,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/.tmp/C/72b31fc27c0b4a30ab629298c92b8c50 as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/C/72b31fc27c0b4a30ab629298c92b8c50 2024-11-18T20:13:15,786 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/C/72b31fc27c0b4a30ab629298c92b8c50, entries=150, sequenceid=249, filesize=11.9 K 2024-11-18T20:13:15,787 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=120.76 KB/123660 for 294b8f086325976977089df95da38d70 in 1247ms, sequenceid=249, compaction requested=false 2024-11-18T20:13:15,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2603): Flush status journal for 294b8f086325976977089df95da38d70: 2024-11-18T20:13:15,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70. 2024-11-18T20:13:15,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=73 2024-11-18T20:13:15,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.HMaster(4169): Remote procedure done, pid=73 2024-11-18T20:13:15,789 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=73, resume processing ppid=72 2024-11-18T20:13:15,790 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=73, ppid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4010 sec 2024-11-18T20:13:15,793 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees in 1.4060 sec 2024-11-18T20:13:15,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(8855): Flush requested on 294b8f086325976977089df95da38d70 2024-11-18T20:13:15,795 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-18T20:13:15,795 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 294b8f086325976977089df95da38d70 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-18T20:13:15,800 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/.tmp/A/c292d643f1b9494d97c748f09bb89345 is 50, key is test_row_0/A:col10/1731960794648/Put/seqid=0 2024-11-18T20:13:15,809 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:15,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54614 deadline: 1731960855805, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:15,810 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:15,810 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:15,810 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:13:15,812 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:15,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54602 deadline: 1731960855808, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:15,812 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:15,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54600 deadline: 1731960855808, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:15,812 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:15,812 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:15,812 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:13:15,813 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:15,813 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:15,813 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:13:15,813 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:15,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54532 deadline: 1731960855810, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:15,814 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:15,814 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:15,814 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:15,814 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:13:15,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54570 deadline: 1731960855811, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:15,814 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:15,814 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:15,814 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:13:15,819 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742116_1292 (size=12301) 2024-11-18T20:13:15,915 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:15,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54614 deadline: 1731960855912, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:15,915 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:15,916 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:15,916 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:13:15,926 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:15,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54602 deadline: 1731960855923, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:15,927 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:15,927 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:15,927 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:15,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54600 deadline: 1731960855923, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:15,927 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:13:15,927 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:15,927 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:15,927 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:15,927 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:15,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54532 deadline: 1731960855923, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:15,927 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:13:15,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54570 deadline: 1731960855923, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:15,927 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:15,927 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:15,927 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:13:15,927 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:15,927 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:15,928 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:13:16,125 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:16,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54614 deadline: 1731960856122, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:16,125 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:16,125 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:16,125 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:13:16,135 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:16,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54602 deadline: 1731960856133, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:16,135 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:16,135 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:16,135 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:16,135 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:13:16,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54600 deadline: 1731960856133, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:16,136 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:16,136 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:16,136 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:16,136 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:13:16,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54532 deadline: 1731960856133, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:16,136 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:16,136 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:16,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54570 deadline: 1731960856133, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:16,136 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:16,136 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:13:16,136 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:16,136 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:16,136 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:13:16,219 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=274 (bloomFilter=true), to=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/.tmp/A/c292d643f1b9494d97c748f09bb89345 2024-11-18T20:13:16,226 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/.tmp/B/463dcbcaead9413a88c7640e95132d29 is 50, key is test_row_0/B:col10/1731960794648/Put/seqid=0 2024-11-18T20:13:16,229 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742117_1293 (size=12301) 2024-11-18T20:13:16,435 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:16,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54614 deadline: 1731960856433, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:16,436 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:16,436 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:16,436 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:13:16,446 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:16,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54602 deadline: 1731960856443, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:16,446 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:16,446 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:16,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54600 deadline: 1731960856443, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:16,446 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:16,446 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:13:16,446 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:16,446 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:16,446 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:16,446 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:13:16,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54570 deadline: 1731960856444, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:16,447 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:16,447 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:16,447 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:16,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54532 deadline: 1731960856444, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:16,447 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:13:16,447 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:16,447 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:16,447 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:13:16,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-18T20:13:16,543 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-18T20:13:16,543 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-18T20:13:16,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] procedure2.ProcedureExecutor(1139): Stored pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees 2024-11-18T20:13:16,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-18T20:13:16,545 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-18T20:13:16,545 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-18T20:13:16,545 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=75, ppid=74, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-18T20:13:16,630 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=274 (bloomFilter=true), to=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/.tmp/B/463dcbcaead9413a88c7640e95132d29 2024-11-18T20:13:16,637 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/.tmp/C/5bcb2e7a514845e1a7115244e78fb5d0 is 50, key is test_row_0/C:col10/1731960794648/Put/seqid=0 2024-11-18T20:13:16,640 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742118_1294 (size=12301) 2024-11-18T20:13:16,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-18T20:13:16,697 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35963 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-18T20:13:16,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70. 2024-11-18T20:13:16,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70. as already flushing 2024-11-18T20:13:16,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70. 2024-11-18T20:13:16,698 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 294b8f086325976977089df95da38d70, NAME => 'TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:13:16,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 294b8f086325976977089df95da38d70, NAME => 'TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:13:16,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 294b8f086325976977089df95da38d70, NAME => 'TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 294b8f086325976977089df95da38d70, NAME => 'TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:13:16,851 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35963 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-18T20:13:16,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70. 2024-11-18T20:13:16,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70. as already flushing 2024-11-18T20:13:16,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70. 2024-11-18T20:13:16,852 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 294b8f086325976977089df95da38d70, NAME => 'TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:13:16,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 294b8f086325976977089df95da38d70, NAME => 'TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:13:16,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 294b8f086325976977089df95da38d70, NAME => 'TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 294b8f086325976977089df95da38d70, NAME => 'TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:13:16,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-18T20:13:16,945 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:16,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54614 deadline: 1731960856943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:16,946 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:16,946 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:16,946 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:13:16,955 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:16,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54602 deadline: 1731960856953, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:16,956 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:16,956 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:16,956 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:13:16,956 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:16,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54600 deadline: 1731960856953, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:16,956 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:16,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54570 deadline: 1731960856953, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:16,956 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:16,956 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:16,956 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:16,956 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:13:16,956 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:16,956 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:16,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54532 deadline: 1731960856954, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:16,956 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:13:16,956 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:16,956 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:16,956 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:13:17,004 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35963 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-18T20:13:17,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70. 2024-11-18T20:13:17,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70. as already flushing 2024-11-18T20:13:17,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70. 2024-11-18T20:13:17,004 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 294b8f086325976977089df95da38d70, NAME => 'TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:13:17,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 294b8f086325976977089df95da38d70, NAME => 'TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:13:17,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 294b8f086325976977089df95da38d70, NAME => 'TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 294b8f086325976977089df95da38d70, NAME => 'TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:13:17,042 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=274 (bloomFilter=true), to=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/.tmp/C/5bcb2e7a514845e1a7115244e78fb5d0 2024-11-18T20:13:17,049 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/.tmp/A/c292d643f1b9494d97c748f09bb89345 as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/A/c292d643f1b9494d97c748f09bb89345 2024-11-18T20:13:17,053 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/A/c292d643f1b9494d97c748f09bb89345, entries=150, sequenceid=274, filesize=12.0 K 2024-11-18T20:13:17,054 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/.tmp/B/463dcbcaead9413a88c7640e95132d29 as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/B/463dcbcaead9413a88c7640e95132d29 2024-11-18T20:13:17,059 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/B/463dcbcaead9413a88c7640e95132d29, entries=150, sequenceid=274, filesize=12.0 K 2024-11-18T20:13:17,060 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/.tmp/C/5bcb2e7a514845e1a7115244e78fb5d0 as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/C/5bcb2e7a514845e1a7115244e78fb5d0 2024-11-18T20:13:17,068 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/C/5bcb2e7a514845e1a7115244e78fb5d0, entries=150, sequenceid=274, filesize=12.0 K 2024-11-18T20:13:17,069 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=73.80 KB/75570 for 294b8f086325976977089df95da38d70 in 1274ms, sequenceid=274, compaction requested=true 2024-11-18T20:13:17,069 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 294b8f086325976977089df95da38d70: 2024-11-18T20:13:17,069 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 294b8f086325976977089df95da38d70:A, priority=-2147483648, current under compaction store size is 1 2024-11-18T20:13:17,069 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:13:17,069 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T20:13:17,069 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 294b8f086325976977089df95da38d70:B, priority=-2147483648, current under compaction store size is 2 2024-11-18T20:13:17,069 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T20:13:17,069 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 294b8f086325976977089df95da38d70:C, priority=-2147483648, current under compaction store size is 3 2024-11-18T20:13:17,069 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-18T20:13:17,069 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T20:13:17,070 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37115 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T20:13:17,070 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HStore(1541): 294b8f086325976977089df95da38d70/A is initiating minor compaction (all files) 2024-11-18T20:13:17,070 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 294b8f086325976977089df95da38d70/A in TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70. 2024-11-18T20:13:17,070 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/A/f336b89ef6f440c5ae78966493d3e3ec, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/A/6bb29e1da64e40c0845d3a90926ee4ca, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/A/c292d643f1b9494d97c748f09bb89345] into tmpdir=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/.tmp, totalSize=36.2 K 2024-11-18T20:13:17,071 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37115 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T20:13:17,071 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HStore(1541): 294b8f086325976977089df95da38d70/B is initiating minor compaction (all files) 2024-11-18T20:13:17,071 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 294b8f086325976977089df95da38d70/B in TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70. 2024-11-18T20:13:17,071 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/B/20466a25c4c74ded8a242b3da0aaa76a, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/B/aa4a5d754cfc4a819020fe51384a849f, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/B/463dcbcaead9413a88c7640e95132d29] into tmpdir=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/.tmp, totalSize=36.2 K 2024-11-18T20:13:17,071 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] compactions.Compactor(225): Compacting f336b89ef6f440c5ae78966493d3e3ec, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=233, earliestPutTs=1731960792808 2024-11-18T20:13:17,072 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.Compactor(225): Compacting 20466a25c4c74ded8a242b3da0aaa76a, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=233, earliestPutTs=1731960792808 2024-11-18T20:13:17,072 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6bb29e1da64e40c0845d3a90926ee4ca, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=249, earliestPutTs=1731960793969 2024-11-18T20:13:17,072 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.Compactor(225): Compacting aa4a5d754cfc4a819020fe51384a849f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=249, earliestPutTs=1731960793969 2024-11-18T20:13:17,072 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] compactions.Compactor(225): Compacting c292d643f1b9494d97c748f09bb89345, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=274, earliestPutTs=1731960794648 2024-11-18T20:13:17,072 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.Compactor(225): Compacting 463dcbcaead9413a88c7640e95132d29, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=274, earliestPutTs=1731960794648 2024-11-18T20:13:17,081 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 294b8f086325976977089df95da38d70#A#compaction#251 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:13:17,081 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/.tmp/A/747c2a99a3c446768093ceb49dce01d3 is 50, key is test_row_0/A:col10/1731960794648/Put/seqid=0 2024-11-18T20:13:17,083 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 294b8f086325976977089df95da38d70#B#compaction#252 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:13:17,083 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/.tmp/B/6ba2a1c8952147d4be8f837de5c4f49b is 50, key is test_row_0/B:col10/1731960794648/Put/seqid=0 2024-11-18T20:13:17,102 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742119_1295 (size=12915) 2024-11-18T20:13:17,123 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742120_1296 (size=12915) 2024-11-18T20:13:17,133 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/.tmp/B/6ba2a1c8952147d4be8f837de5c4f49b as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/B/6ba2a1c8952147d4be8f837de5c4f49b 2024-11-18T20:13:17,140 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 294b8f086325976977089df95da38d70/B of 294b8f086325976977089df95da38d70 into 6ba2a1c8952147d4be8f837de5c4f49b(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:13:17,141 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 294b8f086325976977089df95da38d70: 2024-11-18T20:13:17,141 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., storeName=294b8f086325976977089df95da38d70/B, priority=13, startTime=1731960797069; duration=0sec 2024-11-18T20:13:17,141 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T20:13:17,141 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 294b8f086325976977089df95da38d70:B 2024-11-18T20:13:17,141 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T20:13:17,143 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37115 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T20:13:17,143 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HStore(1541): 294b8f086325976977089df95da38d70/C is initiating minor compaction (all files) 2024-11-18T20:13:17,143 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 294b8f086325976977089df95da38d70/C in TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70. 2024-11-18T20:13:17,143 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/C/61247116172143a5a44b686ab32303af, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/C/72b31fc27c0b4a30ab629298c92b8c50, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/C/5bcb2e7a514845e1a7115244e78fb5d0] into tmpdir=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/.tmp, totalSize=36.2 K 2024-11-18T20:13:17,144 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.Compactor(225): Compacting 61247116172143a5a44b686ab32303af, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=233, earliestPutTs=1731960792808 2024-11-18T20:13:17,144 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.Compactor(225): Compacting 72b31fc27c0b4a30ab629298c92b8c50, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=249, earliestPutTs=1731960793969 2024-11-18T20:13:17,146 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.Compactor(225): Compacting 5bcb2e7a514845e1a7115244e78fb5d0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=274, earliestPutTs=1731960794648 2024-11-18T20:13:17,156 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 294b8f086325976977089df95da38d70#C#compaction#253 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:13:17,157 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/.tmp/C/a8114ff7c0a340bb9c8248907401d3f2 is 50, key is test_row_0/C:col10/1731960794648/Put/seqid=0 2024-11-18T20:13:17,157 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35963 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-18T20:13:17,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70. 2024-11-18T20:13:17,158 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2902): Flushing 294b8f086325976977089df95da38d70 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-18T20:13:17,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-18T20:13:17,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/.tmp/A/9c4dfcac8a4e4a1c91db376f25e1744f is 50, key is test_row_0/A:col10/1731960795805/Put/seqid=0 2024-11-18T20:13:17,191 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742121_1297 (size=12915) 2024-11-18T20:13:17,193 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742122_1298 (size=12301) 2024-11-18T20:13:17,194 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=289 (bloomFilter=true), to=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/.tmp/A/9c4dfcac8a4e4a1c91db376f25e1744f 2024-11-18T20:13:17,206 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/.tmp/B/0ee038e7421a48c5898b8c84d946f0a6 is 50, key is test_row_0/B:col10/1731960795805/Put/seqid=0 2024-11-18T20:13:17,228 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742123_1299 (size=12301) 2024-11-18T20:13:17,229 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=289 (bloomFilter=true), to=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/.tmp/B/0ee038e7421a48c5898b8c84d946f0a6 2024-11-18T20:13:17,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/.tmp/C/d8d24908329c4a24bb0be3fc4dd78562 is 50, key is test_row_0/C:col10/1731960795805/Put/seqid=0 2024-11-18T20:13:17,264 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742124_1300 (size=12301) 2024-11-18T20:13:17,266 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=289 (bloomFilter=true), to=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/.tmp/C/d8d24908329c4a24bb0be3fc4dd78562 2024-11-18T20:13:17,277 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/.tmp/A/9c4dfcac8a4e4a1c91db376f25e1744f as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/A/9c4dfcac8a4e4a1c91db376f25e1744f 2024-11-18T20:13:17,284 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/A/9c4dfcac8a4e4a1c91db376f25e1744f, entries=150, sequenceid=289, filesize=12.0 K 2024-11-18T20:13:17,287 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/.tmp/B/0ee038e7421a48c5898b8c84d946f0a6 as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/B/0ee038e7421a48c5898b8c84d946f0a6 2024-11-18T20:13:17,292 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/B/0ee038e7421a48c5898b8c84d946f0a6, entries=150, sequenceid=289, filesize=12.0 K 2024-11-18T20:13:17,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/.tmp/C/d8d24908329c4a24bb0be3fc4dd78562 as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/C/d8d24908329c4a24bb0be3fc4dd78562 2024-11-18T20:13:17,298 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/C/d8d24908329c4a24bb0be3fc4dd78562, entries=150, sequenceid=289, filesize=12.0 K 2024-11-18T20:13:17,299 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=0 B/0 for 294b8f086325976977089df95da38d70 in 141ms, sequenceid=289, compaction requested=false 2024-11-18T20:13:17,299 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2603): Flush status journal for 294b8f086325976977089df95da38d70: 2024-11-18T20:13:17,299 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70. 2024-11-18T20:13:17,299 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=75 2024-11-18T20:13:17,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.HMaster(4169): Remote procedure done, pid=75 2024-11-18T20:13:17,304 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=75, resume processing ppid=74 2024-11-18T20:13:17,304 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=75, ppid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 756 msec 2024-11-18T20:13:17,307 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees in 762 msec 2024-11-18T20:13:17,507 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/.tmp/A/747c2a99a3c446768093ceb49dce01d3 as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/A/747c2a99a3c446768093ceb49dce01d3 2024-11-18T20:13:17,511 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 294b8f086325976977089df95da38d70/A of 294b8f086325976977089df95da38d70 into 747c2a99a3c446768093ceb49dce01d3(size=12.6 K), total size for store is 24.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:13:17,512 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 294b8f086325976977089df95da38d70: 2024-11-18T20:13:17,512 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., storeName=294b8f086325976977089df95da38d70/A, priority=13, startTime=1731960797069; duration=0sec 2024-11-18T20:13:17,512 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:13:17,512 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 294b8f086325976977089df95da38d70:A 2024-11-18T20:13:17,598 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/.tmp/C/a8114ff7c0a340bb9c8248907401d3f2 as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/C/a8114ff7c0a340bb9c8248907401d3f2 2024-11-18T20:13:17,602 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 294b8f086325976977089df95da38d70/C of 294b8f086325976977089df95da38d70 into a8114ff7c0a340bb9c8248907401d3f2(size=12.6 K), total size for store is 24.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:13:17,602 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 294b8f086325976977089df95da38d70: 2024-11-18T20:13:17,603 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., storeName=294b8f086325976977089df95da38d70/C, priority=13, startTime=1731960797069; duration=0sec 2024-11-18T20:13:17,603 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:13:17,603 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 294b8f086325976977089df95da38d70:C 2024-11-18T20:13:17,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-18T20:13:17,683 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-18T20:13:17,684 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-18T20:13:17,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] procedure2.ProcedureExecutor(1139): Stored pid=76, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees 2024-11-18T20:13:17,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-18T20:13:17,686 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=76, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-18T20:13:17,686 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=76, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-18T20:13:17,687 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=77, ppid=76, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-18T20:13:17,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-18T20:13:17,839 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35963 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-18T20:13:17,839 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70. 2024-11-18T20:13:17,839 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2603): Flush status journal for 294b8f086325976977089df95da38d70: 2024-11-18T20:13:17,839 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70. 2024-11-18T20:13:17,839 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=77 2024-11-18T20:13:17,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.HMaster(4169): Remote procedure done, pid=77 2024-11-18T20:13:17,843 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=77, resume processing ppid=76 2024-11-18T20:13:17,843 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=77, ppid=76, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 155 msec 2024-11-18T20:13:17,844 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=76, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees in 159 msec 2024-11-18T20:13:17,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(8855): Flush requested on 294b8f086325976977089df95da38d70 2024-11-18T20:13:17,981 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-18T20:13:17,981 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 294b8f086325976977089df95da38d70 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-18T20:13:17,988 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/.tmp/A/7fa7a394f3024177987167f47961229f is 50, key is test_row_0/A:col10/1731960797980/Put/seqid=0 2024-11-18T20:13:18,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-18T20:13:18,003 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-18T20:13:18,004 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-18T20:13:18,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] procedure2.ProcedureExecutor(1139): Stored pid=78, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees 2024-11-18T20:13:18,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-18T20:13:18,006 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=78, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-18T20:13:18,006 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=78, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-18T20:13:18,007 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=79, ppid=78, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-18T20:13:18,011 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:18,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54570 deadline: 1731960858007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:18,012 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:18,012 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:18,012 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:13:18,012 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:18,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54600 deadline: 1731960858009, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:18,012 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:18,013 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:18,013 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:13:18,013 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:18,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54614 deadline: 1731960858010, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:18,014 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:18,014 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:18,014 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:18,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54602 deadline: 1731960858011, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:18,014 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:13:18,014 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:18,014 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:18,014 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:18,014 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:13:18,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54532 deadline: 1731960858012, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:18,014 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:18,014 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:18,014 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:13:18,022 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742125_1301 (size=14741) 2024-11-18T20:13:18,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-18T20:13:18,126 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:18,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54614 deadline: 1731960858124, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:18,126 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:18,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54602 deadline: 1731960858124, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:18,126 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:18,126 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:18,126 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:13:18,127 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:18,127 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:18,127 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:18,127 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:13:18,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54532 deadline: 1731960858124, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:18,127 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:18,127 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:18,127 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:18,127 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:13:18,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54570 deadline: 1731960858124, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:18,127 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:18,127 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:18,127 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:18,127 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:13:18,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54600 deadline: 1731960858124, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:18,127 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:18,128 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:18,128 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:13:18,160 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35963 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-18T20:13:18,160 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70. 2024-11-18T20:13:18,160 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70. as already flushing 2024-11-18T20:13:18,160 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70. 2024-11-18T20:13:18,160 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => 294b8f086325976977089df95da38d70, NAME => 'TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:13:18,160 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => 294b8f086325976977089df95da38d70, NAME => 'TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:13:18,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 294b8f086325976977089df95da38d70, NAME => 'TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 294b8f086325976977089df95da38d70, NAME => 'TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:13:18,313 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35963 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-18T20:13:18,313 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70. 2024-11-18T20:13:18,313 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70. as already flushing 2024-11-18T20:13:18,313 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70. 2024-11-18T20:13:18,313 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => 294b8f086325976977089df95da38d70, NAME => 'TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:13:18,313 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => 294b8f086325976977089df95da38d70, NAME => 'TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:13:18,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 294b8f086325976977089df95da38d70, NAME => 'TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 294b8f086325976977089df95da38d70, NAME => 'TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:13:18,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-18T20:13:18,340 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:18,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54614 deadline: 1731960858337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:18,340 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:18,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54570 deadline: 1731960858337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:18,340 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:18,340 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:18,340 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:13:18,340 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:18,340 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:18,340 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:18,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54532 deadline: 1731960858337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:18,340 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:13:18,341 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:18,341 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:18,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54600 deadline: 1731960858338, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:18,341 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:18,341 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:13:18,341 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:18,341 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:18,341 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:18,341 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:13:18,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54602 deadline: 1731960858341, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:18,341 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:18,341 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:18,341 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:13:18,423 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=304 (bloomFilter=true), to=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/.tmp/A/7fa7a394f3024177987167f47961229f 2024-11-18T20:13:18,432 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/.tmp/B/a3cc67c317944dad81a6c4637b0bb49e is 50, key is test_row_0/B:col10/1731960797980/Put/seqid=0 2024-11-18T20:13:18,439 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742126_1302 (size=12301) 2024-11-18T20:13:18,440 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=304 (bloomFilter=true), to=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/.tmp/B/a3cc67c317944dad81a6c4637b0bb49e 2024-11-18T20:13:18,449 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/.tmp/C/6d36c0ed0f0e416681d6b49cc458d365 is 50, key is test_row_0/C:col10/1731960797980/Put/seqid=0 2024-11-18T20:13:18,464 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742127_1303 (size=12301) 2024-11-18T20:13:18,464 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=304 (bloomFilter=true), to=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/.tmp/C/6d36c0ed0f0e416681d6b49cc458d365 2024-11-18T20:13:18,466 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35963 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-18T20:13:18,466 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70. 2024-11-18T20:13:18,466 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70. as already flushing 2024-11-18T20:13:18,466 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70. 2024-11-18T20:13:18,466 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => 294b8f086325976977089df95da38d70, NAME => 'TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:13:18,467 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => 294b8f086325976977089df95da38d70, NAME => 'TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:13:18,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 294b8f086325976977089df95da38d70, NAME => 'TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 294b8f086325976977089df95da38d70, NAME => 'TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:13:18,477 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/.tmp/A/7fa7a394f3024177987167f47961229f as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/A/7fa7a394f3024177987167f47961229f 2024-11-18T20:13:18,492 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/A/7fa7a394f3024177987167f47961229f, entries=200, sequenceid=304, filesize=14.4 K 2024-11-18T20:13:18,493 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/.tmp/B/a3cc67c317944dad81a6c4637b0bb49e as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/B/a3cc67c317944dad81a6c4637b0bb49e 2024-11-18T20:13:18,497 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/B/a3cc67c317944dad81a6c4637b0bb49e, entries=150, sequenceid=304, filesize=12.0 K 2024-11-18T20:13:18,498 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/.tmp/C/6d36c0ed0f0e416681d6b49cc458d365 as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/C/6d36c0ed0f0e416681d6b49cc458d365 2024-11-18T20:13:18,503 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/C/6d36c0ed0f0e416681d6b49cc458d365, entries=150, sequenceid=304, filesize=12.0 K 2024-11-18T20:13:18,505 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 294b8f086325976977089df95da38d70 in 524ms, sequenceid=304, compaction requested=true 2024-11-18T20:13:18,505 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 294b8f086325976977089df95da38d70: 2024-11-18T20:13:18,505 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 294b8f086325976977089df95da38d70:A, priority=-2147483648, current under compaction store size is 1 2024-11-18T20:13:18,505 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:13:18,505 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 294b8f086325976977089df95da38d70:B, priority=-2147483648, current under compaction store size is 2 2024-11-18T20:13:18,505 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T20:13:18,505 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:13:18,505 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T20:13:18,505 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 294b8f086325976977089df95da38d70:C, priority=-2147483648, current under compaction store size is 3 2024-11-18T20:13:18,505 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T20:13:18,506 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37517 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T20:13:18,506 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39957 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T20:13:18,506 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HStore(1541): 294b8f086325976977089df95da38d70/B is initiating minor compaction (all files) 2024-11-18T20:13:18,506 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HStore(1541): 294b8f086325976977089df95da38d70/A is initiating minor compaction (all files) 2024-11-18T20:13:18,506 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 294b8f086325976977089df95da38d70/B in TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70. 2024-11-18T20:13:18,506 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 294b8f086325976977089df95da38d70/A in TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70. 2024-11-18T20:13:18,506 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/A/747c2a99a3c446768093ceb49dce01d3, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/A/9c4dfcac8a4e4a1c91db376f25e1744f, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/A/7fa7a394f3024177987167f47961229f] into tmpdir=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/.tmp, totalSize=39.0 K 2024-11-18T20:13:18,506 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/B/6ba2a1c8952147d4be8f837de5c4f49b, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/B/0ee038e7421a48c5898b8c84d946f0a6, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/B/a3cc67c317944dad81a6c4637b0bb49e] into tmpdir=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/.tmp, totalSize=36.6 K 2024-11-18T20:13:18,507 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] compactions.Compactor(225): Compacting 747c2a99a3c446768093ceb49dce01d3, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=274, earliestPutTs=1731960794648 2024-11-18T20:13:18,507 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.Compactor(225): Compacting 6ba2a1c8952147d4be8f837de5c4f49b, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=274, earliestPutTs=1731960794648 2024-11-18T20:13:18,508 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9c4dfcac8a4e4a1c91db376f25e1744f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=289, earliestPutTs=1731960795802 2024-11-18T20:13:18,508 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.Compactor(225): Compacting 0ee038e7421a48c5898b8c84d946f0a6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=289, earliestPutTs=1731960795802 2024-11-18T20:13:18,508 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7fa7a394f3024177987167f47961229f, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=304, earliestPutTs=1731960797979 2024-11-18T20:13:18,508 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.Compactor(225): Compacting a3cc67c317944dad81a6c4637b0bb49e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=304, earliestPutTs=1731960797980 2024-11-18T20:13:18,517 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 294b8f086325976977089df95da38d70#A#compaction#260 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:13:18,517 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/.tmp/A/4e7248aaa27746e885ad8d2a9fee9d72 is 50, key is test_row_0/A:col10/1731960797980/Put/seqid=0 2024-11-18T20:13:18,519 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 294b8f086325976977089df95da38d70#B#compaction#261 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:13:18,519 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/.tmp/B/1b167d039f734ca5bc88725e2fb064c2 is 50, key is test_row_0/B:col10/1731960797980/Put/seqid=0 2024-11-18T20:13:18,528 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742128_1304 (size=13017) 2024-11-18T20:13:18,531 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742129_1305 (size=13017) 2024-11-18T20:13:18,536 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/.tmp/A/4e7248aaa27746e885ad8d2a9fee9d72 as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/A/4e7248aaa27746e885ad8d2a9fee9d72 2024-11-18T20:13:18,540 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 294b8f086325976977089df95da38d70/A of 294b8f086325976977089df95da38d70 into 4e7248aaa27746e885ad8d2a9fee9d72(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:13:18,540 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 294b8f086325976977089df95da38d70: 2024-11-18T20:13:18,540 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., storeName=294b8f086325976977089df95da38d70/A, priority=13, startTime=1731960798505; duration=0sec 2024-11-18T20:13:18,540 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T20:13:18,540 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 294b8f086325976977089df95da38d70:A 2024-11-18T20:13:18,541 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T20:13:18,541 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37517 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T20:13:18,541 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HStore(1541): 294b8f086325976977089df95da38d70/C is initiating minor compaction (all files) 2024-11-18T20:13:18,541 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 294b8f086325976977089df95da38d70/C in TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70. 2024-11-18T20:13:18,541 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/C/a8114ff7c0a340bb9c8248907401d3f2, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/C/d8d24908329c4a24bb0be3fc4dd78562, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/C/6d36c0ed0f0e416681d6b49cc458d365] into tmpdir=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/.tmp, totalSize=36.6 K 2024-11-18T20:13:18,542 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] compactions.Compactor(225): Compacting a8114ff7c0a340bb9c8248907401d3f2, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=274, earliestPutTs=1731960794648 2024-11-18T20:13:18,542 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] compactions.Compactor(225): Compacting d8d24908329c4a24bb0be3fc4dd78562, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=289, earliestPutTs=1731960795802 2024-11-18T20:13:18,543 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6d36c0ed0f0e416681d6b49cc458d365, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=304, earliestPutTs=1731960797980 2024-11-18T20:13:18,553 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 294b8f086325976977089df95da38d70#C#compaction#262 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:13:18,554 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/.tmp/C/176916338d184fbcbc1d29112054c92e is 50, key is test_row_0/C:col10/1731960797980/Put/seqid=0 2024-11-18T20:13:18,559 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742130_1306 (size=13017) 2024-11-18T20:13:18,621 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35963 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-18T20:13:18,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70. 2024-11-18T20:13:18,622 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2902): Flushing 294b8f086325976977089df95da38d70 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-18T20:13:18,627 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/.tmp/A/af73e07d2a5e4cdc9ec2ad846af20659 is 50, key is test_row_0/A:col10/1731960798009/Put/seqid=0 2024-11-18T20:13:18,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-18T20:13:18,645 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70. as already flushing 2024-11-18T20:13:18,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(8855): Flush requested on 294b8f086325976977089df95da38d70 2024-11-18T20:13:18,647 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742131_1307 (size=12301) 2024-11-18T20:13:18,648 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=329 (bloomFilter=true), to=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/.tmp/A/af73e07d2a5e4cdc9ec2ad846af20659 2024-11-18T20:13:18,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/.tmp/B/ffe8eb5fb24d447b8879e8f869a754df is 50, key is test_row_0/B:col10/1731960798009/Put/seqid=0 2024-11-18T20:13:18,664 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:18,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54614 deadline: 1731960858659, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:18,665 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:18,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54570 deadline: 1731960858659, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:18,665 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:18,665 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:18,665 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:13:18,666 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:18,666 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:18,666 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:13:18,666 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:18,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54532 deadline: 1731960858661, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:18,667 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:18,667 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:18,667 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:13:18,667 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:18,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54602 deadline: 1731960858662, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:18,667 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:18,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54600 deadline: 1731960858664, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:18,668 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:18,668 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:18,668 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:13:18,668 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:18,668 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:18,668 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:13:18,695 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742132_1308 (size=12301) 2024-11-18T20:13:18,696 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=329 (bloomFilter=true), to=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/.tmp/B/ffe8eb5fb24d447b8879e8f869a754df 2024-11-18T20:13:18,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/.tmp/C/4f1afef087b747c186a2e24869c30aae is 50, key is test_row_0/C:col10/1731960798009/Put/seqid=0 2024-11-18T20:13:18,735 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742133_1309 (size=12301) 2024-11-18T20:13:18,736 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=329 (bloomFilter=true), to=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/.tmp/C/4f1afef087b747c186a2e24869c30aae 2024-11-18T20:13:18,743 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/.tmp/A/af73e07d2a5e4cdc9ec2ad846af20659 as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/A/af73e07d2a5e4cdc9ec2ad846af20659 2024-11-18T20:13:18,749 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/A/af73e07d2a5e4cdc9ec2ad846af20659, entries=150, sequenceid=329, filesize=12.0 K 2024-11-18T20:13:18,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/.tmp/B/ffe8eb5fb24d447b8879e8f869a754df as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/B/ffe8eb5fb24d447b8879e8f869a754df 2024-11-18T20:13:18,758 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/B/ffe8eb5fb24d447b8879e8f869a754df, entries=150, sequenceid=329, filesize=12.0 K 2024-11-18T20:13:18,759 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/.tmp/C/4f1afef087b747c186a2e24869c30aae as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/C/4f1afef087b747c186a2e24869c30aae 2024-11-18T20:13:18,766 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/C/4f1afef087b747c186a2e24869c30aae, entries=150, sequenceid=329, filesize=12.0 K 2024-11-18T20:13:18,767 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=67.09 KB/68700 for 294b8f086325976977089df95da38d70 in 146ms, sequenceid=329, compaction requested=false 2024-11-18T20:13:18,767 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2603): Flush status journal for 294b8f086325976977089df95da38d70: 2024-11-18T20:13:18,767 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70. 2024-11-18T20:13:18,767 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=79 2024-11-18T20:13:18,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.HMaster(4169): Remote procedure done, pid=79 2024-11-18T20:13:18,775 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=79, resume processing ppid=78 2024-11-18T20:13:18,775 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=79, ppid=78, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 763 msec 2024-11-18T20:13:18,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(8855): Flush requested on 294b8f086325976977089df95da38d70 2024-11-18T20:13:18,776 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-18T20:13:18,776 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 294b8f086325976977089df95da38d70 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-18T20:13:18,777 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=78, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees in 772 msec 2024-11-18T20:13:18,783 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/.tmp/A/d0938223c5274d37af2a9768626ce5a0 is 50, key is test_row_0/A:col10/1731960798659/Put/seqid=0 2024-11-18T20:13:18,790 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742134_1310 (size=17181) 2024-11-18T20:13:18,791 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=344 (bloomFilter=true), to=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/.tmp/A/d0938223c5274d37af2a9768626ce5a0 2024-11-18T20:13:18,799 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/.tmp/B/0bcb9c97a66a432797639ad418e14c0d is 50, key is test_row_0/B:col10/1731960798659/Put/seqid=0 2024-11-18T20:13:18,811 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:18,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54602 deadline: 1731960858804, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:18,811 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:18,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54570 deadline: 1731960858805, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:18,812 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:18,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54600 deadline: 1731960858805, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:18,812 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:18,812 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:18,812 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:18,812 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:13:18,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54614 deadline: 1731960858807, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:18,812 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:18,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54532 deadline: 1731960858807, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:18,812 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:18,812 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:18,812 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:13:18,813 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:18,813 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:18,813 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:18,813 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:13:18,813 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:18,813 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:13:18,813 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:18,813 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:18,813 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:13:18,822 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742135_1311 (size=12301) 2024-11-18T20:13:18,823 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=344 (bloomFilter=true), to=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/.tmp/B/0bcb9c97a66a432797639ad418e14c0d 2024-11-18T20:13:18,855 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/.tmp/C/392a0ab7d9dd41c69c2bc6315bb86699 is 50, key is test_row_0/C:col10/1731960798659/Put/seqid=0 2024-11-18T20:13:18,866 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742136_1312 (size=12301) 2024-11-18T20:13:18,867 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=344 (bloomFilter=true), to=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/.tmp/C/392a0ab7d9dd41c69c2bc6315bb86699 2024-11-18T20:13:18,871 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/.tmp/A/d0938223c5274d37af2a9768626ce5a0 as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/A/d0938223c5274d37af2a9768626ce5a0 2024-11-18T20:13:18,877 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/A/d0938223c5274d37af2a9768626ce5a0, entries=250, sequenceid=344, filesize=16.8 K 2024-11-18T20:13:18,878 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/.tmp/B/0bcb9c97a66a432797639ad418e14c0d as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/B/0bcb9c97a66a432797639ad418e14c0d 2024-11-18T20:13:18,882 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/B/0bcb9c97a66a432797639ad418e14c0d, entries=150, sequenceid=344, filesize=12.0 K 2024-11-18T20:13:18,883 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/.tmp/C/392a0ab7d9dd41c69c2bc6315bb86699 as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/C/392a0ab7d9dd41c69c2bc6315bb86699 2024-11-18T20:13:18,889 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/C/392a0ab7d9dd41c69c2bc6315bb86699, entries=150, sequenceid=344, filesize=12.0 K 2024-11-18T20:13:18,892 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=120.76 KB/123660 for 294b8f086325976977089df95da38d70 in 115ms, sequenceid=344, compaction requested=true 2024-11-18T20:13:18,892 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 294b8f086325976977089df95da38d70: 2024-11-18T20:13:18,892 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 294b8f086325976977089df95da38d70:A, priority=-2147483648, current under compaction store size is 3 2024-11-18T20:13:18,892 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T20:13:18,892 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 294b8f086325976977089df95da38d70:B, priority=-2147483648, current under compaction store size is 3 2024-11-18T20:13:18,892 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-18T20:13:18,892 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 294b8f086325976977089df95da38d70:C, priority=-2147483648, current under compaction store size is 3 2024-11-18T20:13:18,892 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-11-18T20:13:18,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(8855): Flush requested on 294b8f086325976977089df95da38d70 2024-11-18T20:13:18,927 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-18T20:13:18,927 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 294b8f086325976977089df95da38d70 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-18T20:13:18,933 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/.tmp/A/e4c4076a5e8e4364ae727617b7d236af is 50, key is test_row_0/A:col10/1731960798805/Put/seqid=0 2024-11-18T20:13:18,938 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/.tmp/B/1b167d039f734ca5bc88725e2fb064c2 as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/B/1b167d039f734ca5bc88725e2fb064c2 2024-11-18T20:13:18,944 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 294b8f086325976977089df95da38d70/B of 294b8f086325976977089df95da38d70 into 1b167d039f734ca5bc88725e2fb064c2(size=12.7 K), total size for store is 36.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:13:18,944 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 294b8f086325976977089df95da38d70: 2024-11-18T20:13:18,944 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., storeName=294b8f086325976977089df95da38d70/B, priority=13, startTime=1731960798505; duration=0sec 2024-11-18T20:13:18,945 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-11-18T20:13:18,945 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:18,945 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:18,945 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 294b8f086325976977089df95da38d70:B 2024-11-18T20:13:18,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54600 deadline: 1731960858938, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:18,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54614 deadline: 1731960858937, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:18,945 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:18,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54532 deadline: 1731960858939, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:18,946 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:18,946 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:18,946 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:13:18,946 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:18,946 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:18,946 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:13:18,946 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 294b8f086325976977089df95da38d70:B 2024-11-18T20:13:18,947 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:18,947 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:18,947 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 3 compacting, 2 eligible, 16 blocking 2024-11-18T20:13:18,947 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:13:18,947 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-18T20:13:18,947 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-18T20:13:18,947 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70. because compaction request was cancelled 2024-11-18T20:13:18,947 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 294b8f086325976977089df95da38d70:C 2024-11-18T20:13:18,948 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T20:13:18,948 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T20:13:18,948 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HStore(1541): 294b8f086325976977089df95da38d70/B is initiating minor compaction (all files) 2024-11-18T20:13:18,948 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 294b8f086325976977089df95da38d70/B in TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70. 2024-11-18T20:13:18,948 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/B/1b167d039f734ca5bc88725e2fb064c2, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/B/ffe8eb5fb24d447b8879e8f869a754df, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/B/0bcb9c97a66a432797639ad418e14c0d] into tmpdir=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/.tmp, totalSize=36.7 K 2024-11-18T20:13:18,949 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.Compactor(225): Compacting 1b167d039f734ca5bc88725e2fb064c2, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=304, earliestPutTs=1731960797980 2024-11-18T20:13:18,949 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.Compactor(225): Compacting ffe8eb5fb24d447b8879e8f869a754df, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=329, earliestPutTs=1731960797999 2024-11-18T20:13:18,949 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.Compactor(225): Compacting 0bcb9c97a66a432797639ad418e14c0d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=344, earliestPutTs=1731960798657 2024-11-18T20:13:18,951 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:18,951 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:18,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54602 deadline: 1731960858945, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:18,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54570 deadline: 1731960858946, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:18,952 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:18,952 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:18,952 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:13:18,952 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:18,952 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:18,952 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:13:18,954 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742137_1313 (size=12301) 2024-11-18T20:13:18,955 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=367 (bloomFilter=true), to=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/.tmp/A/e4c4076a5e8e4364ae727617b7d236af 2024-11-18T20:13:18,959 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 294b8f086325976977089df95da38d70#B#compaction#270 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:13:18,959 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/.tmp/B/b67a4da182ba43058d11d8a7d8b1ab4c is 50, key is test_row_0/B:col10/1731960798659/Put/seqid=0 2024-11-18T20:13:18,969 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/.tmp/C/176916338d184fbcbc1d29112054c92e as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/C/176916338d184fbcbc1d29112054c92e 2024-11-18T20:13:18,976 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 294b8f086325976977089df95da38d70/C of 294b8f086325976977089df95da38d70 into 176916338d184fbcbc1d29112054c92e(size=12.7 K), total size for store is 36.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:13:18,976 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 294b8f086325976977089df95da38d70: 2024-11-18T20:13:18,976 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., storeName=294b8f086325976977089df95da38d70/C, priority=13, startTime=1731960798505; duration=0sec 2024-11-18T20:13:18,976 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T20:13:18,976 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 294b8f086325976977089df95da38d70:C 2024-11-18T20:13:18,976 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T20:13:18,977 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 42499 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T20:13:18,977 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HStore(1541): 294b8f086325976977089df95da38d70/A is initiating minor compaction (all files) 2024-11-18T20:13:18,977 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 294b8f086325976977089df95da38d70/A in TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70. 2024-11-18T20:13:18,977 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/A/4e7248aaa27746e885ad8d2a9fee9d72, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/A/af73e07d2a5e4cdc9ec2ad846af20659, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/A/d0938223c5274d37af2a9768626ce5a0] into tmpdir=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/.tmp, totalSize=41.5 K 2024-11-18T20:13:18,977 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4e7248aaa27746e885ad8d2a9fee9d72, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=304, earliestPutTs=1731960797980 2024-11-18T20:13:18,979 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/.tmp/B/0e309084e07e42cfbbb801c25cd9b6f4 is 50, key is test_row_0/B:col10/1731960798805/Put/seqid=0 2024-11-18T20:13:18,979 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742138_1314 (size=13119) 2024-11-18T20:13:18,979 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] compactions.Compactor(225): Compacting af73e07d2a5e4cdc9ec2ad846af20659, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=329, earliestPutTs=1731960797999 2024-11-18T20:13:18,980 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] compactions.Compactor(225): Compacting d0938223c5274d37af2a9768626ce5a0, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=344, earliestPutTs=1731960798657 2024-11-18T20:13:18,984 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/.tmp/B/b67a4da182ba43058d11d8a7d8b1ab4c as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/B/b67a4da182ba43058d11d8a7d8b1ab4c 2024-11-18T20:13:18,988 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 294b8f086325976977089df95da38d70/B of 294b8f086325976977089df95da38d70 into b67a4da182ba43058d11d8a7d8b1ab4c(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:13:18,988 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 294b8f086325976977089df95da38d70: 2024-11-18T20:13:18,988 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., storeName=294b8f086325976977089df95da38d70/B, priority=13, startTime=1731960798892; duration=0sec 2024-11-18T20:13:18,989 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:13:18,989 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 294b8f086325976977089df95da38d70:B 2024-11-18T20:13:18,995 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 294b8f086325976977089df95da38d70#A#compaction#272 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:13:18,995 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/.tmp/A/021c6f5e56764333892e460b10b4634f is 50, key is test_row_0/A:col10/1731960798659/Put/seqid=0 2024-11-18T20:13:19,019 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742140_1316 (size=13119) 2024-11-18T20:13:19,026 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742139_1315 (size=12301) 2024-11-18T20:13:19,055 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:19,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54614 deadline: 1731960859053, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:19,055 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:19,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54532 deadline: 1731960859053, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:19,055 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:19,055 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:19,056 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:13:19,056 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:19,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54600 deadline: 1731960859053, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:19,056 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:19,056 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:19,056 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:13:19,056 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:19,056 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:19,056 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:13:19,066 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:19,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54570 deadline: 1731960859063, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:19,066 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:19,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54602 deadline: 1731960859063, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:19,066 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:19,067 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:19,067 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:13:19,067 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:19,067 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:19,067 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:13:19,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-18T20:13:19,143 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-18T20:13:19,144 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-18T20:13:19,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] procedure2.ProcedureExecutor(1139): Stored pid=80, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees 2024-11-18T20:13:19,145 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=80, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-18T20:13:19,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-18T20:13:19,146 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=80, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-18T20:13:19,146 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=81, ppid=80, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-18T20:13:19,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-18T20:13:19,266 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:19,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54614 deadline: 1731960859263, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:19,267 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:19,267 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:19,267 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:19,267 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:13:19,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54532 deadline: 1731960859263, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:19,267 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:19,267 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:19,267 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:19,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54600 deadline: 1731960859263, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:19,267 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:13:19,267 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:19,267 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:19,267 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:13:19,276 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:19,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54570 deadline: 1731960859273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:19,277 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:19,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54602 deadline: 1731960859273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:19,277 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:19,277 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:19,277 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:13:19,277 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:19,277 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:19,277 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:13:19,299 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35963 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-18T20:13:19,299 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70. 2024-11-18T20:13:19,299 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70. as already flushing 2024-11-18T20:13:19,299 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70. 2024-11-18T20:13:19,299 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => 294b8f086325976977089df95da38d70, NAME => 'TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:13:19,299 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => 294b8f086325976977089df95da38d70, NAME => 'TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:13:19,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 294b8f086325976977089df95da38d70, NAME => 'TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 294b8f086325976977089df95da38d70, NAME => 'TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:13:19,423 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/.tmp/A/021c6f5e56764333892e460b10b4634f as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/A/021c6f5e56764333892e460b10b4634f 2024-11-18T20:13:19,427 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=367 (bloomFilter=true), to=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/.tmp/B/0e309084e07e42cfbbb801c25cd9b6f4 2024-11-18T20:13:19,428 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 294b8f086325976977089df95da38d70/A of 294b8f086325976977089df95da38d70 into 021c6f5e56764333892e460b10b4634f(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:13:19,428 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 294b8f086325976977089df95da38d70: 2024-11-18T20:13:19,428 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., storeName=294b8f086325976977089df95da38d70/A, priority=13, startTime=1731960798892; duration=0sec 2024-11-18T20:13:19,428 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:13:19,428 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 294b8f086325976977089df95da38d70:A 2024-11-18T20:13:19,435 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/.tmp/C/ae10870a09014e12a2f809b9199e133b is 50, key is test_row_0/C:col10/1731960798805/Put/seqid=0 2024-11-18T20:13:19,438 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742141_1317 (size=12301) 2024-11-18T20:13:19,451 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35963 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-18T20:13:19,451 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70. 2024-11-18T20:13:19,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70. as already flushing 2024-11-18T20:13:19,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70. 2024-11-18T20:13:19,452 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => 294b8f086325976977089df95da38d70, NAME => 'TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:13:19,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => 294b8f086325976977089df95da38d70, NAME => 'TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:13:19,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 294b8f086325976977089df95da38d70, NAME => 'TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 294b8f086325976977089df95da38d70, NAME => 'TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:13:19,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-18T20:13:19,576 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:19,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54614 deadline: 1731960859573, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:19,576 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:19,576 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:19,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54532 deadline: 1731960859574, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:19,576 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:19,576 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:13:19,576 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:19,576 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:19,576 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:19,577 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:13:19,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54600 deadline: 1731960859574, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:19,577 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:19,577 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:19,577 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:13:19,583 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:19,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54570 deadline: 1731960859583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:19,584 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:19,584 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:19,584 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:19,584 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:13:19,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54602 deadline: 1731960859583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:19,584 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:19,584 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:19,584 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:13:19,604 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35963 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-18T20:13:19,604 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70. 2024-11-18T20:13:19,604 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70. as already flushing 2024-11-18T20:13:19,604 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70. 2024-11-18T20:13:19,604 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => 294b8f086325976977089df95da38d70, NAME => 'TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:13:19,605 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => 294b8f086325976977089df95da38d70, NAME => 'TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:13:19,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 294b8f086325976977089df95da38d70, NAME => 'TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 294b8f086325976977089df95da38d70, NAME => 'TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:13:19,757 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35963 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-18T20:13:19,757 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70. 2024-11-18T20:13:19,757 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70. as already flushing 2024-11-18T20:13:19,757 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70. 2024-11-18T20:13:19,757 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => 294b8f086325976977089df95da38d70, NAME => 'TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:13:19,757 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => 294b8f086325976977089df95da38d70, NAME => 'TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:13:19,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 294b8f086325976977089df95da38d70, NAME => 'TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 294b8f086325976977089df95da38d70, NAME => 'TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:13:19,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-18T20:13:19,838 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=367 (bloomFilter=true), to=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/.tmp/C/ae10870a09014e12a2f809b9199e133b 2024-11-18T20:13:19,849 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/.tmp/A/e4c4076a5e8e4364ae727617b7d236af as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/A/e4c4076a5e8e4364ae727617b7d236af 2024-11-18T20:13:19,853 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/A/e4c4076a5e8e4364ae727617b7d236af, entries=150, sequenceid=367, filesize=12.0 K 2024-11-18T20:13:19,854 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/.tmp/B/0e309084e07e42cfbbb801c25cd9b6f4 as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/B/0e309084e07e42cfbbb801c25cd9b6f4 2024-11-18T20:13:19,861 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/B/0e309084e07e42cfbbb801c25cd9b6f4, entries=150, sequenceid=367, filesize=12.0 K 2024-11-18T20:13:19,862 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/.tmp/C/ae10870a09014e12a2f809b9199e133b as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/C/ae10870a09014e12a2f809b9199e133b 2024-11-18T20:13:19,867 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/C/ae10870a09014e12a2f809b9199e133b, entries=150, sequenceid=367, filesize=12.0 K 2024-11-18T20:13:19,867 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for 294b8f086325976977089df95da38d70 in 940ms, sequenceid=367, compaction requested=true 2024-11-18T20:13:19,868 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 294b8f086325976977089df95da38d70: 2024-11-18T20:13:19,868 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 294b8f086325976977089df95da38d70:A, priority=-2147483648, current under compaction store size is 1 2024-11-18T20:13:19,868 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:13:19,868 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 294b8f086325976977089df95da38d70:B, priority=-2147483648, current under compaction store size is 2 2024-11-18T20:13:19,868 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:13:19,868 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-18T20:13:19,868 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 294b8f086325976977089df95da38d70:C, priority=-2147483648, current under compaction store size is 3 2024-11-18T20:13:19,868 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-18T20:13:19,868 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T20:13:19,868 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-18T20:13:19,868 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-18T20:13:19,868 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-18T20:13:19,868 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-18T20:13:19,868 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70. because compaction request was cancelled 2024-11-18T20:13:19,868 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70. because compaction request was cancelled 2024-11-18T20:13:19,868 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 294b8f086325976977089df95da38d70:B 2024-11-18T20:13:19,869 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 294b8f086325976977089df95da38d70:A 2024-11-18T20:13:19,869 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-18T20:13:19,870 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49920 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-18T20:13:19,870 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HStore(1541): 294b8f086325976977089df95da38d70/C is initiating minor compaction (all files) 2024-11-18T20:13:19,870 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 294b8f086325976977089df95da38d70/C in TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70. 2024-11-18T20:13:19,870 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/C/176916338d184fbcbc1d29112054c92e, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/C/4f1afef087b747c186a2e24869c30aae, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/C/392a0ab7d9dd41c69c2bc6315bb86699, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/C/ae10870a09014e12a2f809b9199e133b] into tmpdir=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/.tmp, totalSize=48.8 K 2024-11-18T20:13:19,871 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.Compactor(225): Compacting 176916338d184fbcbc1d29112054c92e, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=304, earliestPutTs=1731960797980 2024-11-18T20:13:19,871 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.Compactor(225): Compacting 4f1afef087b747c186a2e24869c30aae, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=329, earliestPutTs=1731960797999 2024-11-18T20:13:19,871 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.Compactor(225): Compacting 392a0ab7d9dd41c69c2bc6315bb86699, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=344, earliestPutTs=1731960798657 2024-11-18T20:13:19,872 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.Compactor(225): Compacting ae10870a09014e12a2f809b9199e133b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=367, earliestPutTs=1731960798805 2024-11-18T20:13:19,882 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 294b8f086325976977089df95da38d70#C#compaction#274 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:13:19,883 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/.tmp/C/b525a90460c443438dc7cfbb263f1740 is 50, key is test_row_0/C:col10/1731960798805/Put/seqid=0 2024-11-18T20:13:19,896 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742142_1318 (size=13153) 2024-11-18T20:13:19,902 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/.tmp/C/b525a90460c443438dc7cfbb263f1740 as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/C/b525a90460c443438dc7cfbb263f1740 2024-11-18T20:13:19,909 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 294b8f086325976977089df95da38d70/C of 294b8f086325976977089df95da38d70 into b525a90460c443438dc7cfbb263f1740(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:13:19,909 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 294b8f086325976977089df95da38d70: 2024-11-18T20:13:19,909 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., storeName=294b8f086325976977089df95da38d70/C, priority=12, startTime=1731960799868; duration=0sec 2024-11-18T20:13:19,909 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:13:19,909 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 294b8f086325976977089df95da38d70:C 2024-11-18T20:13:19,912 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35963 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-18T20:13:19,912 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70. 2024-11-18T20:13:19,912 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2902): Flushing 294b8f086325976977089df95da38d70 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-18T20:13:19,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/.tmp/A/4c703a84861d4fc4acee2fad108ac3ad is 50, key is test_row_0/A:col10/1731960798943/Put/seqid=0 2024-11-18T20:13:19,950 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742143_1319 (size=12301) 2024-11-18T20:13:20,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(8855): Flush requested on 294b8f086325976977089df95da38d70 2024-11-18T20:13:20,085 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70. as already flushing 2024-11-18T20:13:20,144 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:20,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54614 deadline: 1731960860138, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:20,144 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:20,144 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:20,144 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:20,144 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:13:20,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54602 deadline: 1731960860138, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:20,144 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:20,145 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:20,144 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:20,145 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:13:20,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54600 deadline: 1731960860138, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:20,145 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:20,145 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:20,145 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:13:20,145 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:20,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54532 deadline: 1731960860139, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:20,145 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:20,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54570 deadline: 1731960860140, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:20,145 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:20,145 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:20,145 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:13:20,145 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:20,145 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:20,146 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:13:20,256 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:20,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54602 deadline: 1731960860253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:20,256 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:20,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54600 deadline: 1731960860253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:20,256 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:20,256 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:20,256 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:20,257 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:13:20,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54570 deadline: 1731960860253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:20,257 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:20,257 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:20,257 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:20,257 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:13:20,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54532 deadline: 1731960860253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:20,257 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:20,257 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:20,257 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:20,257 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:13:20,257 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:20,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54614 deadline: 1731960860254, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:20,257 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:20,257 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:13:20,257 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:20,257 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:20,257 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:13:20,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-18T20:13:20,351 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=385 (bloomFilter=true), to=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/.tmp/A/4c703a84861d4fc4acee2fad108ac3ad 2024-11-18T20:13:20,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/.tmp/B/17ce2b3d10444368b72559b9ca9c6747 is 50, key is test_row_0/B:col10/1731960798943/Put/seqid=0 2024-11-18T20:13:20,364 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742144_1320 (size=12301) 2024-11-18T20:13:20,365 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=385 (bloomFilter=true), to=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/.tmp/B/17ce2b3d10444368b72559b9ca9c6747 2024-11-18T20:13:20,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/.tmp/C/5c93d491f68d441586bc4655ea82b564 is 50, key is test_row_0/C:col10/1731960798943/Put/seqid=0 2024-11-18T20:13:20,381 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742145_1321 (size=12301) 2024-11-18T20:13:20,466 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:20,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54602 deadline: 1731960860463, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:20,466 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:20,466 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:20,467 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:20,467 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:13:20,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54600 deadline: 1731960860464, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:20,467 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:20,467 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:20,467 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:20,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54570 deadline: 1731960860464, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:20,467 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:13:20,467 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:20,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54532 deadline: 1731960860464, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:20,467 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:20,467 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:20,467 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:13:20,467 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:20,467 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:20,467 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:20,467 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:13:20,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54614 deadline: 1731960860464, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:20,468 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:20,468 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:20,468 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:13:20,775 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:20,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54602 deadline: 1731960860773, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:20,775 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:20,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54600 deadline: 1731960860773, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:20,775 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:20,775 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:20,775 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:13:20,775 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:20,775 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:20,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54532 deadline: 1731960860774, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:20,775 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:20,775 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:13:20,775 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:20,775 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:20,776 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:20,776 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:13:20,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54570 deadline: 1731960860774, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:20,776 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:20,776 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:20,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54614 deadline: 1731960860774, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:20,776 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:20,776 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:13:20,776 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:20,776 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:20,776 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:13:20,781 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=385 (bloomFilter=true), to=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/.tmp/C/5c93d491f68d441586bc4655ea82b564 2024-11-18T20:13:20,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/.tmp/A/4c703a84861d4fc4acee2fad108ac3ad as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/A/4c703a84861d4fc4acee2fad108ac3ad 2024-11-18T20:13:20,790 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/A/4c703a84861d4fc4acee2fad108ac3ad, entries=150, sequenceid=385, filesize=12.0 K 2024-11-18T20:13:20,790 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/.tmp/B/17ce2b3d10444368b72559b9ca9c6747 as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/B/17ce2b3d10444368b72559b9ca9c6747 2024-11-18T20:13:20,794 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/B/17ce2b3d10444368b72559b9ca9c6747, entries=150, sequenceid=385, filesize=12.0 K 2024-11-18T20:13:20,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/.tmp/C/5c93d491f68d441586bc4655ea82b564 as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/C/5c93d491f68d441586bc4655ea82b564 2024-11-18T20:13:20,799 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/C/5c93d491f68d441586bc4655ea82b564, entries=150, sequenceid=385, filesize=12.0 K 2024-11-18T20:13:20,800 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for 294b8f086325976977089df95da38d70 in 888ms, sequenceid=385, compaction requested=true 2024-11-18T20:13:20,800 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2603): Flush status journal for 294b8f086325976977089df95da38d70: 2024-11-18T20:13:20,800 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70. 2024-11-18T20:13:20,800 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=81 2024-11-18T20:13:20,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.HMaster(4169): Remote procedure done, pid=81 2024-11-18T20:13:20,803 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=81, resume processing ppid=80 2024-11-18T20:13:20,803 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=81, ppid=80, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6550 sec 2024-11-18T20:13:20,805 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=80, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees in 1.6590 sec 2024-11-18T20:13:21,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(8855): Flush requested on 294b8f086325976977089df95da38d70 2024-11-18T20:13:21,286 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-18T20:13:21,286 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 294b8f086325976977089df95da38d70 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-18T20:13:21,291 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/.tmp/A/1a5c886fee254cacb50b53c46d86168f is 50, key is test_row_0/A:col10/1731960801284/Put/seqid=0 2024-11-18T20:13:21,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-18T20:13:21,293 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-18T20:13:21,295 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-18T20:13:21,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] procedure2.ProcedureExecutor(1139): Stored pid=82, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees 2024-11-18T20:13:21,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-18T20:13:21,297 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=82, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-18T20:13:21,298 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=82, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-18T20:13:21,298 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=83, ppid=82, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-18T20:13:21,306 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742146_1322 (size=12301) 2024-11-18T20:13:21,326 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:21,326 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:21,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54600 deadline: 1731960861293, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:21,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54570 deadline: 1731960861294, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:21,327 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:21,327 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:21,327 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:21,327 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:21,327 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:13:21,327 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:13:21,330 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:21,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54532 deadline: 1731960861327, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:21,331 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:21,331 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:21,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54602 deadline: 1731960861327, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:21,331 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:21,331 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:13:21,331 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:21,331 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:21,331 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:21,331 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:13:21,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54614 deadline: 1731960861327, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:21,331 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:21,331 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:21,331 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:13:21,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-18T20:13:21,436 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:21,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54570 deadline: 1731960861433, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:21,436 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:21,436 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:21,436 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:13:21,436 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:21,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54532 deadline: 1731960861433, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:21,437 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:21,437 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:21,437 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:13:21,437 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:21,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54602 deadline: 1731960861433, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:21,437 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:21,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54600 deadline: 1731960861433, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:21,437 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:21,437 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:21,437 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:21,437 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:13:21,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54614 deadline: 1731960861433, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:21,437 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:21,437 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:21,437 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:13:21,444 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:21,444 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:21,444 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:13:21,450 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35963 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-11-18T20:13:21,450 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70. 2024-11-18T20:13:21,450 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70. as already flushing 2024-11-18T20:13:21,450 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70. 2024-11-18T20:13:21,450 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] handler.RSProcedureHandler(58): pid=83 java.io.IOException: Unable to complete flush {ENCODED => 294b8f086325976977089df95da38d70, NAME => 'TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:13:21,450 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=83 java.io.IOException: Unable to complete flush {ENCODED => 294b8f086325976977089df95da38d70, NAME => 'TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:13:21,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.HMaster(4177): Remote procedure failed, pid=83 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 294b8f086325976977089df95da38d70, NAME => 'TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 294b8f086325976977089df95da38d70, NAME => 'TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:13:21,603 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35963 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-11-18T20:13:21,603 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70. 2024-11-18T20:13:21,604 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70. as already flushing 2024-11-18T20:13:21,604 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70. 2024-11-18T20:13:21,604 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] handler.RSProcedureHandler(58): pid=83 java.io.IOException: Unable to complete flush {ENCODED => 294b8f086325976977089df95da38d70, NAME => 'TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:13:21,604 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=83 java.io.IOException: Unable to complete flush {ENCODED => 294b8f086325976977089df95da38d70, NAME => 'TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:13:21,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.HMaster(4177): Remote procedure failed, pid=83 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 294b8f086325976977089df95da38d70, NAME => 'TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 294b8f086325976977089df95da38d70, NAME => 'TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:13:21,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-18T20:13:21,646 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:21,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54532 deadline: 1731960861643, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:21,647 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:21,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54600 deadline: 1731960861643, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:21,647 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:21,647 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:21,647 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:21,647 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:13:21,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54570 deadline: 1731960861643, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:21,647 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:21,647 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:21,647 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:13:21,647 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:21,647 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:21,647 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:13:21,647 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:21,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54602 deadline: 1731960861644, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:21,648 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:21,648 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:21,648 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:13:21,656 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:21,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54614 deadline: 1731960861653, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:21,656 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:21,656 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:21,656 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:13:21,707 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=409 (bloomFilter=true), to=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/.tmp/A/1a5c886fee254cacb50b53c46d86168f 2024-11-18T20:13:21,717 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/.tmp/B/264fb1f7d1b342eb850231cb31a51abc is 50, key is test_row_0/B:col10/1731960801284/Put/seqid=0 2024-11-18T20:13:21,720 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742147_1323 (size=12301) 2024-11-18T20:13:21,720 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=409 (bloomFilter=true), to=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/.tmp/B/264fb1f7d1b342eb850231cb31a51abc 2024-11-18T20:13:21,727 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/.tmp/C/ad5b9ef5a15e40339947529315a8b274 is 50, key is test_row_0/C:col10/1731960801284/Put/seqid=0 2024-11-18T20:13:21,731 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742148_1324 (size=12301) 2024-11-18T20:13:21,732 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=409 (bloomFilter=true), to=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/.tmp/C/ad5b9ef5a15e40339947529315a8b274 2024-11-18T20:13:21,737 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/.tmp/A/1a5c886fee254cacb50b53c46d86168f as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/A/1a5c886fee254cacb50b53c46d86168f 2024-11-18T20:13:21,742 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/A/1a5c886fee254cacb50b53c46d86168f, entries=150, sequenceid=409, filesize=12.0 K 2024-11-18T20:13:21,743 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/.tmp/B/264fb1f7d1b342eb850231cb31a51abc as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/B/264fb1f7d1b342eb850231cb31a51abc 2024-11-18T20:13:21,747 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/B/264fb1f7d1b342eb850231cb31a51abc, entries=150, sequenceid=409, filesize=12.0 K 2024-11-18T20:13:21,747 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/.tmp/C/ad5b9ef5a15e40339947529315a8b274 as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/C/ad5b9ef5a15e40339947529315a8b274 2024-11-18T20:13:21,752 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/C/ad5b9ef5a15e40339947529315a8b274, entries=150, sequenceid=409, filesize=12.0 K 2024-11-18T20:13:21,753 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=67.09 KB/68700 for 294b8f086325976977089df95da38d70 in 467ms, sequenceid=409, compaction requested=true 2024-11-18T20:13:21,753 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 294b8f086325976977089df95da38d70: 2024-11-18T20:13:21,753 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 294b8f086325976977089df95da38d70:A, priority=-2147483648, current under compaction store size is 1 2024-11-18T20:13:21,753 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:13:21,753 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-18T20:13:21,753 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-18T20:13:21,753 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 294b8f086325976977089df95da38d70:B, priority=-2147483648, current under compaction store size is 2 2024-11-18T20:13:21,753 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:13:21,754 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 294b8f086325976977089df95da38d70:C, priority=-2147483648, current under compaction store size is 3 2024-11-18T20:13:21,754 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T20:13:21,756 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50022 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-18T20:13:21,756 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HStore(1541): 294b8f086325976977089df95da38d70/B is initiating minor compaction (all files) 2024-11-18T20:13:21,756 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 294b8f086325976977089df95da38d70/B in TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70. 2024-11-18T20:13:21,756 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/B/b67a4da182ba43058d11d8a7d8b1ab4c, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/B/0e309084e07e42cfbbb801c25cd9b6f4, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/B/17ce2b3d10444368b72559b9ca9c6747, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/B/264fb1f7d1b342eb850231cb31a51abc] into tmpdir=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/.tmp, totalSize=48.8 K 2024-11-18T20:13:21,756 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50022 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-18T20:13:21,757 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HStore(1541): 294b8f086325976977089df95da38d70/A is initiating minor compaction (all files) 2024-11-18T20:13:21,757 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 294b8f086325976977089df95da38d70/A in TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70. 2024-11-18T20:13:21,757 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/A/021c6f5e56764333892e460b10b4634f, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/A/e4c4076a5e8e4364ae727617b7d236af, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/A/4c703a84861d4fc4acee2fad108ac3ad, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/A/1a5c886fee254cacb50b53c46d86168f] into tmpdir=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/.tmp, totalSize=48.8 K 2024-11-18T20:13:21,757 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.Compactor(225): Compacting b67a4da182ba43058d11d8a7d8b1ab4c, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=344, earliestPutTs=1731960798657 2024-11-18T20:13:21,757 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] compactions.Compactor(225): Compacting 021c6f5e56764333892e460b10b4634f, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=344, earliestPutTs=1731960798657 2024-11-18T20:13:21,757 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] compactions.Compactor(225): Compacting e4c4076a5e8e4364ae727617b7d236af, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=367, earliestPutTs=1731960798805 2024-11-18T20:13:21,757 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.Compactor(225): Compacting 0e309084e07e42cfbbb801c25cd9b6f4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=367, earliestPutTs=1731960798805 2024-11-18T20:13:21,758 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4c703a84861d4fc4acee2fad108ac3ad, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=385, earliestPutTs=1731960798938 2024-11-18T20:13:21,758 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.Compactor(225): Compacting 17ce2b3d10444368b72559b9ca9c6747, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=385, earliestPutTs=1731960798938 2024-11-18T20:13:21,758 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1a5c886fee254cacb50b53c46d86168f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=409, earliestPutTs=1731960800137 2024-11-18T20:13:21,758 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.Compactor(225): Compacting 264fb1f7d1b342eb850231cb31a51abc, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=409, earliestPutTs=1731960800137 2024-11-18T20:13:21,762 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35963 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-11-18T20:13:21,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70. 2024-11-18T20:13:21,763 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2902): Flushing 294b8f086325976977089df95da38d70 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-18T20:13:21,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/.tmp/A/179955a798a84f0dae1b040d29ae9b26 is 50, key is test_row_0/A:col10/1731960801291/Put/seqid=0 2024-11-18T20:13:21,769 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 294b8f086325976977089df95da38d70#A#compaction#282 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:13:21,770 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/.tmp/A/d038446a3b4e463e8c833e29e20a8c22 is 50, key is test_row_0/A:col10/1731960801284/Put/seqid=0 2024-11-18T20:13:21,772 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 294b8f086325976977089df95da38d70#B#compaction#283 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:13:21,773 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/.tmp/B/1b093495e501499285c4d3b10478b24a is 50, key is test_row_0/B:col10/1731960801284/Put/seqid=0 2024-11-18T20:13:21,787 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742149_1325 (size=12301) 2024-11-18T20:13:21,787 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=422 (bloomFilter=true), to=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/.tmp/A/179955a798a84f0dae1b040d29ae9b26 2024-11-18T20:13:21,790 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742150_1326 (size=13255) 2024-11-18T20:13:21,798 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/.tmp/A/d038446a3b4e463e8c833e29e20a8c22 as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/A/d038446a3b4e463e8c833e29e20a8c22 2024-11-18T20:13:21,806 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 294b8f086325976977089df95da38d70/A of 294b8f086325976977089df95da38d70 into d038446a3b4e463e8c833e29e20a8c22(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:13:21,807 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 294b8f086325976977089df95da38d70: 2024-11-18T20:13:21,807 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., storeName=294b8f086325976977089df95da38d70/A, priority=12, startTime=1731960801753; duration=0sec 2024-11-18T20:13:21,807 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T20:13:21,807 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 294b8f086325976977089df95da38d70:A 2024-11-18T20:13:21,807 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T20:13:21,809 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37755 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T20:13:21,809 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HStore(1541): 294b8f086325976977089df95da38d70/C is initiating minor compaction (all files) 2024-11-18T20:13:21,809 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 294b8f086325976977089df95da38d70/C in TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70. 2024-11-18T20:13:21,809 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/C/b525a90460c443438dc7cfbb263f1740, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/C/5c93d491f68d441586bc4655ea82b564, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/C/ad5b9ef5a15e40339947529315a8b274] into tmpdir=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/.tmp, totalSize=36.9 K 2024-11-18T20:13:21,810 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] compactions.Compactor(225): Compacting b525a90460c443438dc7cfbb263f1740, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=367, earliestPutTs=1731960798805 2024-11-18T20:13:21,810 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5c93d491f68d441586bc4655ea82b564, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=385, earliestPutTs=1731960798938 2024-11-18T20:13:21,811 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742151_1327 (size=13255) 2024-11-18T20:13:21,811 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] compactions.Compactor(225): Compacting ad5b9ef5a15e40339947529315a8b274, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=409, earliestPutTs=1731960800137 2024-11-18T20:13:21,819 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/.tmp/B/e99e524319614518b88617e4e8dd15c6 is 50, key is test_row_0/B:col10/1731960801291/Put/seqid=0 2024-11-18T20:13:21,819 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/.tmp/B/1b093495e501499285c4d3b10478b24a as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/B/1b093495e501499285c4d3b10478b24a 2024-11-18T20:13:21,826 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 294b8f086325976977089df95da38d70/B of 294b8f086325976977089df95da38d70 into 1b093495e501499285c4d3b10478b24a(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:13:21,826 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 294b8f086325976977089df95da38d70: 2024-11-18T20:13:21,826 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., storeName=294b8f086325976977089df95da38d70/B, priority=12, startTime=1731960801753; duration=0sec 2024-11-18T20:13:21,827 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:13:21,827 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 294b8f086325976977089df95da38d70:B 2024-11-18T20:13:21,830 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 294b8f086325976977089df95da38d70#C#compaction#285 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:13:21,831 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/.tmp/C/f8c18faca8634429bf036470865845b7 is 50, key is test_row_0/C:col10/1731960801284/Put/seqid=0 2024-11-18T20:13:21,834 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742152_1328 (size=12301) 2024-11-18T20:13:21,836 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=422 (bloomFilter=true), to=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/.tmp/B/e99e524319614518b88617e4e8dd15c6 2024-11-18T20:13:21,875 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/.tmp/C/a30540ccd8fc47eaa4191214a6901e31 is 50, key is test_row_0/C:col10/1731960801291/Put/seqid=0 2024-11-18T20:13:21,875 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742153_1329 (size=13255) 2024-11-18T20:13:21,898 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742154_1330 (size=12301) 2024-11-18T20:13:21,899 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=422 (bloomFilter=true), to=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/.tmp/C/a30540ccd8fc47eaa4191214a6901e31 2024-11-18T20:13:21,904 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/.tmp/A/179955a798a84f0dae1b040d29ae9b26 as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/A/179955a798a84f0dae1b040d29ae9b26 2024-11-18T20:13:21,910 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/A/179955a798a84f0dae1b040d29ae9b26, entries=150, sequenceid=422, filesize=12.0 K 2024-11-18T20:13:21,911 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/.tmp/B/e99e524319614518b88617e4e8dd15c6 as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/B/e99e524319614518b88617e4e8dd15c6 2024-11-18T20:13:21,916 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/B/e99e524319614518b88617e4e8dd15c6, entries=150, sequenceid=422, filesize=12.0 K 2024-11-18T20:13:21,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/.tmp/C/a30540ccd8fc47eaa4191214a6901e31 as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/C/a30540ccd8fc47eaa4191214a6901e31 2024-11-18T20:13:21,921 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/C/a30540ccd8fc47eaa4191214a6901e31, entries=150, sequenceid=422, filesize=12.0 K 2024-11-18T20:13:21,922 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=0 B/0 for 294b8f086325976977089df95da38d70 in 160ms, sequenceid=422, compaction requested=false 2024-11-18T20:13:21,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2603): Flush status journal for 294b8f086325976977089df95da38d70: 2024-11-18T20:13:21,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70. 2024-11-18T20:13:21,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=83 2024-11-18T20:13:21,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.HMaster(4169): Remote procedure done, pid=83 2024-11-18T20:13:21,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-18T20:13:21,925 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=83, resume processing ppid=82 2024-11-18T20:13:21,925 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=83, ppid=82, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 625 msec 2024-11-18T20:13:21,927 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=82, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees in 631 msec 2024-11-18T20:13:21,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(8855): Flush requested on 294b8f086325976977089df95da38d70 2024-11-18T20:13:21,962 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-18T20:13:21,962 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 294b8f086325976977089df95da38d70 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-18T20:13:21,966 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/.tmp/A/6f7f946cbe4f444c9861a6d7e31488e3 is 50, key is test_row_0/A:col10/1731960801960/Put/seqid=0 2024-11-18T20:13:21,973 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742155_1331 (size=14741) 2024-11-18T20:13:21,993 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:21,993 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:21,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54532 deadline: 1731960861987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:21,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54570 deadline: 1731960861987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:21,994 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:21,994 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:21,994 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:21,994 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:21,994 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:21,994 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:13:21,994 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:13:21,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54600 deadline: 1731960861988, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:21,994 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:21,994 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:21,994 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:21,994 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:13:21,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54602 deadline: 1731960861991, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:21,994 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:21,994 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:21,994 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:13:21,997 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:21,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54614 deadline: 1731960861994, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:21,997 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:21,997 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:21,997 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:13:22,104 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:22,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54570 deadline: 1731960862102, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:22,105 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:22,105 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:22,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54532 deadline: 1731960862103, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:22,105 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:22,105 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:13:22,105 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:22,105 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:22,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54602 deadline: 1731960862103, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:22,105 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:22,105 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:13:22,105 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:22,105 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:22,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54600 deadline: 1731960862103, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:22,105 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:22,105 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:13:22,105 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:22,105 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:22,105 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:22,105 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:13:22,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54614 deadline: 1731960862103, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:22,106 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:22,106 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:22,106 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:13:22,279 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/.tmp/C/f8c18faca8634429bf036470865845b7 as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/C/f8c18faca8634429bf036470865845b7 2024-11-18T20:13:22,284 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 294b8f086325976977089df95da38d70/C of 294b8f086325976977089df95da38d70 into f8c18faca8634429bf036470865845b7(size=12.9 K), total size for store is 25.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:13:22,284 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 294b8f086325976977089df95da38d70: 2024-11-18T20:13:22,284 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., storeName=294b8f086325976977089df95da38d70/C, priority=13, startTime=1731960801753; duration=0sec 2024-11-18T20:13:22,284 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:13:22,284 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 294b8f086325976977089df95da38d70:C 2024-11-18T20:13:22,316 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:22,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54570 deadline: 1731960862313, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:22,316 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:22,316 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:22,316 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:22,316 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:22,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54532 deadline: 1731960862313, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:22,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54602 deadline: 1731960862314, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:22,316 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:13:22,316 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:22,316 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:22,316 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:22,316 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:22,316 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:13:22,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54600 deadline: 1731960862314, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:22,316 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:22,316 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:13:22,317 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:22,317 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:22,317 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:22,317 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:13:22,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54614 deadline: 1731960862314, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:22,317 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:22,317 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:22,317 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:13:22,374 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=435 (bloomFilter=true), to=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/.tmp/A/6f7f946cbe4f444c9861a6d7e31488e3 2024-11-18T20:13:22,381 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/.tmp/B/37effb6494b54275a2748ec7f1e9b511 is 50, key is test_row_0/B:col10/1731960801960/Put/seqid=0 2024-11-18T20:13:22,384 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742156_1332 (size=12301) 2024-11-18T20:13:22,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-18T20:13:22,433 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-18T20:13:22,433 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-18T20:13:22,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] procedure2.ProcedureExecutor(1139): Stored pid=84, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=84, table=TestAcidGuarantees 2024-11-18T20:13:22,435 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=84, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=84, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-18T20:13:22,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-18T20:13:22,435 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=84, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=84, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-18T20:13:22,435 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=85, ppid=84, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-18T20:13:22,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-18T20:13:22,586 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35963 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=85 2024-11-18T20:13:22,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70. 2024-11-18T20:13:22,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70. as already flushing 2024-11-18T20:13:22,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70. 2024-11-18T20:13:22,587 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] handler.RSProcedureHandler(58): pid=85 java.io.IOException: Unable to complete flush {ENCODED => 294b8f086325976977089df95da38d70, NAME => 'TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:13:22,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=85 java.io.IOException: Unable to complete flush {ENCODED => 294b8f086325976977089df95da38d70, NAME => 'TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:13:22,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.HMaster(4177): Remote procedure failed, pid=85 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 294b8f086325976977089df95da38d70, NAME => 'TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 294b8f086325976977089df95da38d70, NAME => 'TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:13:22,625 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:22,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54570 deadline: 1731960862624, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:22,626 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:22,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54600 deadline: 1731960862624, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:22,626 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:22,626 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:22,626 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:13:22,626 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:22,626 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:22,626 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:13:22,626 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:22,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54614 deadline: 1731960862624, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:22,626 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:22,626 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:22,626 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:13:22,626 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:22,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54532 deadline: 1731960862624, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:22,627 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:22,627 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:22,627 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:13:22,627 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:22,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54602 deadline: 1731960862625, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:22,627 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:22,627 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:22,627 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:13:22,739 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35963 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=85 2024-11-18T20:13:22,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70. 2024-11-18T20:13:22,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70. as already flushing 2024-11-18T20:13:22,740 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70. 2024-11-18T20:13:22,740 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] handler.RSProcedureHandler(58): pid=85 java.io.IOException: Unable to complete flush {ENCODED => 294b8f086325976977089df95da38d70, NAME => 'TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:13:22,740 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=85 java.io.IOException: Unable to complete flush {ENCODED => 294b8f086325976977089df95da38d70, NAME => 'TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:13:22,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.HMaster(4177): Remote procedure failed, pid=85 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 294b8f086325976977089df95da38d70, NAME => 'TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 294b8f086325976977089df95da38d70, NAME => 'TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:13:22,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-18T20:13:22,784 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=435 (bloomFilter=true), to=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/.tmp/B/37effb6494b54275a2748ec7f1e9b511 2024-11-18T20:13:22,790 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/.tmp/C/b4c510965eb2467d96a4cc74ca3d409b is 50, key is test_row_0/C:col10/1731960801960/Put/seqid=0 2024-11-18T20:13:22,800 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742157_1333 (size=12301) 2024-11-18T20:13:22,892 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35963 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=85 2024-11-18T20:13:22,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70. 2024-11-18T20:13:22,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70. as already flushing 2024-11-18T20:13:22,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70. 2024-11-18T20:13:22,892 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] handler.RSProcedureHandler(58): pid=85 java.io.IOException: Unable to complete flush {ENCODED => 294b8f086325976977089df95da38d70, NAME => 'TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:13:22,893 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=85 java.io.IOException: Unable to complete flush {ENCODED => 294b8f086325976977089df95da38d70, NAME => 'TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:13:22,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.HMaster(4177): Remote procedure failed, pid=85 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 294b8f086325976977089df95da38d70, NAME => 'TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 294b8f086325976977089df95da38d70, NAME => 'TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:13:23,046 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35963 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=85 2024-11-18T20:13:23,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70. 2024-11-18T20:13:23,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70. as already flushing 2024-11-18T20:13:23,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70. 2024-11-18T20:13:23,046 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] handler.RSProcedureHandler(58): pid=85 java.io.IOException: Unable to complete flush {ENCODED => 294b8f086325976977089df95da38d70, NAME => 'TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:13:23,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=85 java.io.IOException: Unable to complete flush {ENCODED => 294b8f086325976977089df95da38d70, NAME => 'TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:13:23,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.HMaster(4177): Remote procedure failed, pid=85 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 294b8f086325976977089df95da38d70, NAME => 'TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 294b8f086325976977089df95da38d70, NAME => 'TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:13:23,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-18T20:13:23,136 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:23,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54570 deadline: 1731960863133, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:23,137 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:23,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54600 deadline: 1731960863133, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:23,137 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:23,137 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:23,137 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:13:23,137 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:23,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54614 deadline: 1731960863133, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:23,137 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:23,137 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:23,137 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:23,137 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:13:23,137 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:23,137 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:13:23,138 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:23,138 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:23,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54602 deadline: 1731960863133, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:23,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54532 deadline: 1731960863133, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:23,138 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:23,138 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:23,138 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:13:23,138 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:23,138 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:23,138 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:13:23,198 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35963 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=85 2024-11-18T20:13:23,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70. 2024-11-18T20:13:23,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70. as already flushing 2024-11-18T20:13:23,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70. 2024-11-18T20:13:23,199 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] handler.RSProcedureHandler(58): pid=85 java.io.IOException: Unable to complete flush {ENCODED => 294b8f086325976977089df95da38d70, NAME => 'TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:13:23,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=85 java.io.IOException: Unable to complete flush {ENCODED => 294b8f086325976977089df95da38d70, NAME => 'TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:13:23,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.HMaster(4177): Remote procedure failed, pid=85 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 294b8f086325976977089df95da38d70, NAME => 'TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 294b8f086325976977089df95da38d70, NAME => 'TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:13:23,200 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=435 (bloomFilter=true), to=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/.tmp/C/b4c510965eb2467d96a4cc74ca3d409b 2024-11-18T20:13:23,204 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/.tmp/A/6f7f946cbe4f444c9861a6d7e31488e3 as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/A/6f7f946cbe4f444c9861a6d7e31488e3 2024-11-18T20:13:23,208 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/A/6f7f946cbe4f444c9861a6d7e31488e3, entries=200, sequenceid=435, filesize=14.4 K 2024-11-18T20:13:23,208 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/.tmp/B/37effb6494b54275a2748ec7f1e9b511 as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/B/37effb6494b54275a2748ec7f1e9b511 2024-11-18T20:13:23,212 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/B/37effb6494b54275a2748ec7f1e9b511, entries=150, sequenceid=435, filesize=12.0 K 2024-11-18T20:13:23,213 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/.tmp/C/b4c510965eb2467d96a4cc74ca3d409b as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/C/b4c510965eb2467d96a4cc74ca3d409b 2024-11-18T20:13:23,216 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/C/b4c510965eb2467d96a4cc74ca3d409b, entries=150, sequenceid=435, filesize=12.0 K 2024-11-18T20:13:23,217 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 294b8f086325976977089df95da38d70 in 1255ms, sequenceid=435, compaction requested=true 2024-11-18T20:13:23,217 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 294b8f086325976977089df95da38d70: 2024-11-18T20:13:23,217 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 294b8f086325976977089df95da38d70:A, priority=-2147483648, current under compaction store size is 1 2024-11-18T20:13:23,217 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:13:23,217 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T20:13:23,217 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 294b8f086325976977089df95da38d70:B, priority=-2147483648, current under compaction store size is 2 2024-11-18T20:13:23,217 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:13:23,217 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T20:13:23,217 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 294b8f086325976977089df95da38d70:C, priority=-2147483648, current under compaction store size is 3 2024-11-18T20:13:23,217 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T20:13:23,218 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37857 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T20:13:23,218 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40297 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T20:13:23,218 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HStore(1541): 294b8f086325976977089df95da38d70/A is initiating minor compaction (all files) 2024-11-18T20:13:23,218 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HStore(1541): 294b8f086325976977089df95da38d70/B is initiating minor compaction (all files) 2024-11-18T20:13:23,218 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 294b8f086325976977089df95da38d70/B in TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70. 2024-11-18T20:13:23,218 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 294b8f086325976977089df95da38d70/A in TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70. 2024-11-18T20:13:23,218 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/A/d038446a3b4e463e8c833e29e20a8c22, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/A/179955a798a84f0dae1b040d29ae9b26, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/A/6f7f946cbe4f444c9861a6d7e31488e3] into tmpdir=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/.tmp, totalSize=39.4 K 2024-11-18T20:13:23,218 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/B/1b093495e501499285c4d3b10478b24a, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/B/e99e524319614518b88617e4e8dd15c6, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/B/37effb6494b54275a2748ec7f1e9b511] into tmpdir=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/.tmp, totalSize=37.0 K 2024-11-18T20:13:23,219 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.Compactor(225): Compacting 1b093495e501499285c4d3b10478b24a, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=409, earliestPutTs=1731960800137 2024-11-18T20:13:23,219 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] compactions.Compactor(225): Compacting d038446a3b4e463e8c833e29e20a8c22, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=409, earliestPutTs=1731960800137 2024-11-18T20:13:23,219 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.Compactor(225): Compacting e99e524319614518b88617e4e8dd15c6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=422, earliestPutTs=1731960801291 2024-11-18T20:13:23,219 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] compactions.Compactor(225): Compacting 179955a798a84f0dae1b040d29ae9b26, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=422, earliestPutTs=1731960801291 2024-11-18T20:13:23,219 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.Compactor(225): Compacting 37effb6494b54275a2748ec7f1e9b511, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=435, earliestPutTs=1731960801956 2024-11-18T20:13:23,219 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6f7f946cbe4f444c9861a6d7e31488e3, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=435, earliestPutTs=1731960801956 2024-11-18T20:13:23,225 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 294b8f086325976977089df95da38d70#B#compaction#290 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:13:23,226 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 294b8f086325976977089df95da38d70#A#compaction#291 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:13:23,226 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/.tmp/B/a3475f5601dd4ea6b623b6af15878a2d is 50, key is test_row_0/B:col10/1731960801960/Put/seqid=0 2024-11-18T20:13:23,226 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/.tmp/A/3164aceb482841ee82585018cc73eff1 is 50, key is test_row_0/A:col10/1731960801960/Put/seqid=0 2024-11-18T20:13:23,239 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742158_1334 (size=13357) 2024-11-18T20:13:23,242 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742159_1335 (size=13357) 2024-11-18T20:13:23,351 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35963 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=85 2024-11-18T20:13:23,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70. 2024-11-18T20:13:23,351 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2902): Flushing 294b8f086325976977089df95da38d70 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-18T20:13:23,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/.tmp/A/9d9f10351eb24831a61ed819801929ed is 50, key is test_row_0/A:col10/1731960801985/Put/seqid=0 2024-11-18T20:13:23,358 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742160_1336 (size=12301) 2024-11-18T20:13:23,359 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=461 (bloomFilter=true), to=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/.tmp/A/9d9f10351eb24831a61ed819801929ed 2024-11-18T20:13:23,366 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/.tmp/B/640c6b837e1b4b87adfa28586a3bea5b is 50, key is test_row_0/B:col10/1731960801985/Put/seqid=0 2024-11-18T20:13:23,368 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742161_1337 (size=12301) 2024-11-18T20:13:23,467 INFO [Thread-1130 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1130. 2024-11-18T20:13:23,467 DEBUG [Thread-1130 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-18T20:13:23,467 DEBUG [Thread-1130 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:13:23,467 DEBUG [Thread-1130 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:13:23,467 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T20:13:23,467 INFO [Thread-1138 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1138. 2024-11-18T20:13:23,467 DEBUG [Thread-1138 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-18T20:13:23,467 DEBUG [Thread-1138 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:13:23,467 DEBUG [Thread-1138 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:13:23,467 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T20:13:23,468 INFO [Thread-1134 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1134. 2024-11-18T20:13:23,468 DEBUG [Thread-1134 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-18T20:13:23,468 DEBUG [Thread-1134 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:13:23,468 DEBUG [Thread-1134 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:13:23,468 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T20:13:23,469 INFO [Thread-1136 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1136. 2024-11-18T20:13:23,469 DEBUG [Thread-1136 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-18T20:13:23,469 DEBUG [Thread-1136 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:13:23,469 DEBUG [Thread-1136 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:13:23,469 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T20:13:23,470 INFO [Thread-1132 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1132. 2024-11-18T20:13:23,470 DEBUG [Thread-1132 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-18T20:13:23,470 DEBUG [Thread-1132 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:13:23,470 DEBUG [Thread-1132 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:13:23,470 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T20:13:23,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-18T20:13:23,650 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/.tmp/B/a3475f5601dd4ea6b623b6af15878a2d as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/B/a3475f5601dd4ea6b623b6af15878a2d 2024-11-18T20:13:23,651 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/.tmp/A/3164aceb482841ee82585018cc73eff1 as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/A/3164aceb482841ee82585018cc73eff1 2024-11-18T20:13:23,656 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 294b8f086325976977089df95da38d70/A of 294b8f086325976977089df95da38d70 into 3164aceb482841ee82585018cc73eff1(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:13:23,656 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 294b8f086325976977089df95da38d70/B of 294b8f086325976977089df95da38d70 into a3475f5601dd4ea6b623b6af15878a2d(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:13:23,656 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 294b8f086325976977089df95da38d70: 2024-11-18T20:13:23,656 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., storeName=294b8f086325976977089df95da38d70/A, priority=13, startTime=1731960803217; duration=0sec 2024-11-18T20:13:23,656 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 294b8f086325976977089df95da38d70: 2024-11-18T20:13:23,656 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., storeName=294b8f086325976977089df95da38d70/B, priority=13, startTime=1731960803217; duration=0sec 2024-11-18T20:13:23,656 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T20:13:23,656 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 294b8f086325976977089df95da38d70:A 2024-11-18T20:13:23,656 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:13:23,656 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T20:13:23,657 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 294b8f086325976977089df95da38d70:B 2024-11-18T20:13:23,658 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37857 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T20:13:23,658 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HStore(1541): 294b8f086325976977089df95da38d70/C is initiating minor compaction (all files) 2024-11-18T20:13:23,658 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 294b8f086325976977089df95da38d70/C in TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70. 2024-11-18T20:13:23,658 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/C/f8c18faca8634429bf036470865845b7, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/C/a30540ccd8fc47eaa4191214a6901e31, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/C/b4c510965eb2467d96a4cc74ca3d409b] into tmpdir=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/.tmp, totalSize=37.0 K 2024-11-18T20:13:23,658 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] compactions.Compactor(225): Compacting f8c18faca8634429bf036470865845b7, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=409, earliestPutTs=1731960800137 2024-11-18T20:13:23,659 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] compactions.Compactor(225): Compacting a30540ccd8fc47eaa4191214a6901e31, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=422, earliestPutTs=1731960801291 2024-11-18T20:13:23,659 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] compactions.Compactor(225): Compacting b4c510965eb2467d96a4cc74ca3d409b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=435, earliestPutTs=1731960801956 2024-11-18T20:13:23,665 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 294b8f086325976977089df95da38d70#C#compaction#294 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:13:23,666 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/.tmp/C/2b43689b248e492d8a333a09d88f6acf is 50, key is test_row_0/C:col10/1731960801960/Put/seqid=0 2024-11-18T20:13:23,669 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742162_1338 (size=13357) 2024-11-18T20:13:23,769 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=461 (bloomFilter=true), to=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/.tmp/B/640c6b837e1b4b87adfa28586a3bea5b 2024-11-18T20:13:23,775 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/.tmp/C/caa23bb7c5c54cfba70fb6531012d520 is 50, key is test_row_0/C:col10/1731960801985/Put/seqid=0 2024-11-18T20:13:23,778 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742163_1339 (size=12301) 2024-11-18T20:13:24,079 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/.tmp/C/2b43689b248e492d8a333a09d88f6acf as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/C/2b43689b248e492d8a333a09d88f6acf 2024-11-18T20:13:24,084 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 294b8f086325976977089df95da38d70/C of 294b8f086325976977089df95da38d70 into 2b43689b248e492d8a333a09d88f6acf(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:13:24,084 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 294b8f086325976977089df95da38d70: 2024-11-18T20:13:24,084 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., storeName=294b8f086325976977089df95da38d70/C, priority=13, startTime=1731960803217; duration=0sec 2024-11-18T20:13:24,085 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:13:24,085 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 294b8f086325976977089df95da38d70:C 2024-11-18T20:13:24,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(8855): Flush requested on 294b8f086325976977089df95da38d70 2024-11-18T20:13:24,147 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70. as already flushing 2024-11-18T20:13:24,147 INFO [Thread-1123 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1123. 2024-11-18T20:13:24,147 DEBUG [Thread-1123 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-18T20:13:24,147 DEBUG [Thread-1123 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:13:24,148 INFO [Thread-1121 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1121. 2024-11-18T20:13:24,148 DEBUG [Thread-1123 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:13:24,148 DEBUG [Thread-1121 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-18T20:13:24,148 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T20:13:24,148 DEBUG [Thread-1121 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:13:24,148 INFO [Thread-1119 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1119. 2024-11-18T20:13:24,148 DEBUG [Thread-1119 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-18T20:13:24,148 DEBUG [Thread-1119 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:13:24,148 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T20:13:24,148 DEBUG [Thread-1119 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:13:24,148 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T20:13:24,148 DEBUG [Thread-1121 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:13:24,150 INFO [Thread-1125 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1125. 2024-11-18T20:13:24,150 DEBUG [Thread-1125 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-18T20:13:24,150 DEBUG [Thread-1125 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:13:24,150 DEBUG [Thread-1125 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:13:24,150 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T20:13:24,155 INFO [Thread-1127 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1127. 2024-11-18T20:13:24,155 DEBUG [Thread-1127 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-18T20:13:24,155 DEBUG [Thread-1127 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:13:24,155 DEBUG [Thread-1127 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:13:24,155 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T20:13:24,180 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=461 (bloomFilter=true), to=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/.tmp/C/caa23bb7c5c54cfba70fb6531012d520 2024-11-18T20:13:24,188 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/.tmp/A/9d9f10351eb24831a61ed819801929ed as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/A/9d9f10351eb24831a61ed819801929ed 2024-11-18T20:13:24,192 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/A/9d9f10351eb24831a61ed819801929ed, entries=150, sequenceid=461, filesize=12.0 K 2024-11-18T20:13:24,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/.tmp/B/640c6b837e1b4b87adfa28586a3bea5b as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/B/640c6b837e1b4b87adfa28586a3bea5b 2024-11-18T20:13:24,196 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/B/640c6b837e1b4b87adfa28586a3bea5b, entries=150, sequenceid=461, filesize=12.0 K 2024-11-18T20:13:24,196 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/.tmp/C/caa23bb7c5c54cfba70fb6531012d520 as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/C/caa23bb7c5c54cfba70fb6531012d520 2024-11-18T20:13:24,199 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/C/caa23bb7c5c54cfba70fb6531012d520, entries=150, sequenceid=461, filesize=12.0 K 2024-11-18T20:13:24,200 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=33.54 KB/34350 for 294b8f086325976977089df95da38d70 in 849ms, sequenceid=461, compaction requested=false 2024-11-18T20:13:24,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2603): Flush status journal for 294b8f086325976977089df95da38d70: 2024-11-18T20:13:24,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70. 2024-11-18T20:13:24,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=85 2024-11-18T20:13:24,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.HMaster(4169): Remote procedure done, pid=85 2024-11-18T20:13:24,203 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=85, resume processing ppid=84 2024-11-18T20:13:24,203 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=85, ppid=84, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7660 sec 2024-11-18T20:13:24,205 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=84, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=84, table=TestAcidGuarantees in 1.7710 sec 2024-11-18T20:13:24,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-18T20:13:24,584 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-18T20:13:24,584 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-18T20:13:24,584 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 69 2024-11-18T20:13:24,584 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 72 2024-11-18T20:13:24,584 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 78 2024-11-18T20:13:24,584 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 74 2024-11-18T20:13:24,584 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 75 2024-11-18T20:13:24,584 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-18T20:13:24,584 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3955 2024-11-18T20:13:24,584 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3892 2024-11-18T20:13:24,584 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3875 2024-11-18T20:13:24,585 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3965 2024-11-18T20:13:24,585 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3892 2024-11-18T20:13:24,585 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-18T20:13:24,585 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-18T20:13:24,585 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testGetAtomicity(AcidGuaranteesTestBase.java:107) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-18T20:13:24,585 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:13:24,585 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:13:24,586 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T20:13:24,587 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-18T20:13:24,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] procedure2.ProcedureExecutor(1139): Stored pid=86, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-18T20:13:24,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-18T20:13:24,594 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731960804593"}]},"ts":"1731960804593"} 2024-11-18T20:13:24,596 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-18T20:13:24,596 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-18T20:13:24,597 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=87, ppid=86, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-18T20:13:24,600 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=88, ppid=87, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=294b8f086325976977089df95da38d70, UNASSIGN}] 2024-11-18T20:13:24,600 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=88, ppid=87, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=294b8f086325976977089df95da38d70, UNASSIGN 2024-11-18T20:13:24,601 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=88 updating hbase:meta row=294b8f086325976977089df95da38d70, regionState=CLOSING, regionLocation=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:24,603 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=88, ppid=87, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=294b8f086325976977089df95da38d70, UNASSIGN because future has completed 2024-11-18T20:13:24,603 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-18T20:13:24,603 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=89, ppid=88, state=RUNNABLE, hasLock=false; CloseRegionProcedure 294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223}] 2024-11-18T20:13:24,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-18T20:13:24,756 INFO [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] handler.UnassignRegionHandler(122): Close 294b8f086325976977089df95da38d70 2024-11-18T20:13:24,756 DEBUG [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-18T20:13:24,756 DEBUG [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegion(1722): Closing 294b8f086325976977089df95da38d70, disabling compactions & flushes 2024-11-18T20:13:24,756 INFO [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70. 2024-11-18T20:13:24,756 DEBUG [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70. 2024-11-18T20:13:24,756 DEBUG [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70. after waiting 0 ms 2024-11-18T20:13:24,756 DEBUG [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70. 2024-11-18T20:13:24,757 INFO [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegion(2902): Flushing 294b8f086325976977089df95da38d70 3/3 column families, dataSize=33.54 KB heapSize=88.64 KB 2024-11-18T20:13:24,763 DEBUG [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/.tmp/A/eb47b3e30a6d493b95cb7d84a36a7bef is 50, key is test_row_0/A:col10/1731960804154/Put/seqid=0 2024-11-18T20:13:24,769 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742164_1340 (size=9857) 2024-11-18T20:13:24,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-18T20:13:25,171 INFO [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=472 (bloomFilter=true), to=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/.tmp/A/eb47b3e30a6d493b95cb7d84a36a7bef 2024-11-18T20:13:25,182 DEBUG [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/.tmp/B/3b2d15d05d104c9cb0e2603c0b68496a is 50, key is test_row_0/B:col10/1731960804154/Put/seqid=0 2024-11-18T20:13:25,186 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742165_1341 (size=9857) 2024-11-18T20:13:25,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-18T20:13:25,587 INFO [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=472 (bloomFilter=true), to=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/.tmp/B/3b2d15d05d104c9cb0e2603c0b68496a 2024-11-18T20:13:25,597 DEBUG [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/.tmp/C/146c7cd638ca4a09ba6a72226c880dfc is 50, key is test_row_0/C:col10/1731960804154/Put/seqid=0 2024-11-18T20:13:25,600 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742166_1342 (size=9857) 2024-11-18T20:13:25,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-18T20:13:26,001 INFO [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=472 (bloomFilter=true), to=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/.tmp/C/146c7cd638ca4a09ba6a72226c880dfc 2024-11-18T20:13:26,010 DEBUG [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/.tmp/A/eb47b3e30a6d493b95cb7d84a36a7bef as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/A/eb47b3e30a6d493b95cb7d84a36a7bef 2024-11-18T20:13:26,016 INFO [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/A/eb47b3e30a6d493b95cb7d84a36a7bef, entries=100, sequenceid=472, filesize=9.6 K 2024-11-18T20:13:26,016 DEBUG [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/.tmp/B/3b2d15d05d104c9cb0e2603c0b68496a as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/B/3b2d15d05d104c9cb0e2603c0b68496a 2024-11-18T20:13:26,020 INFO [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/B/3b2d15d05d104c9cb0e2603c0b68496a, entries=100, sequenceid=472, filesize=9.6 K 2024-11-18T20:13:26,021 DEBUG [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/.tmp/C/146c7cd638ca4a09ba6a72226c880dfc as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/C/146c7cd638ca4a09ba6a72226c880dfc 2024-11-18T20:13:26,024 INFO [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/C/146c7cd638ca4a09ba6a72226c880dfc, entries=100, sequenceid=472, filesize=9.6 K 2024-11-18T20:13:26,025 INFO [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for 294b8f086325976977089df95da38d70 in 1269ms, sequenceid=472, compaction requested=true 2024-11-18T20:13:26,025 DEBUG [StoreCloser-TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/A/2f328b8a126d4d1a85c43d2f16bb4d46, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/A/68b7edf47fb44fad8c46066aa8579a85, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/A/3cabbf51719548c8a5de6ec4f94917fc, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/A/88535c1b8c9e420994fb3e68ed8d74d9, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/A/56e41c81598b46ac93b521ec008bc0fb, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/A/e4181afa412d49b98b17bbf3a2c56f2a, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/A/f0a1d5aafb0840f5a1fad13c65d79fbf, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/A/3f0423b700404204bee394167d8140ce, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/A/278c8263546e4e03a8190fffe4c44dde, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/A/f62e78b53c944147aa95cb47c8f64e96, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/A/e53dd6b2568444f391f1c1726107c192, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/A/c40f358a3dac4deca6a5df11adbbef03, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/A/57592715ce4b460db24a60d036c13459, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/A/2ca7ca5abfab41c49390187908b97e7b, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/A/1a2f88f8f9b94234b63337e5b7a1da79, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/A/f336b89ef6f440c5ae78966493d3e3ec, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/A/6bb29e1da64e40c0845d3a90926ee4ca, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/A/747c2a99a3c446768093ceb49dce01d3, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/A/c292d643f1b9494d97c748f09bb89345, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/A/9c4dfcac8a4e4a1c91db376f25e1744f, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/A/7fa7a394f3024177987167f47961229f, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/A/4e7248aaa27746e885ad8d2a9fee9d72, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/A/af73e07d2a5e4cdc9ec2ad846af20659, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/A/d0938223c5274d37af2a9768626ce5a0, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/A/021c6f5e56764333892e460b10b4634f, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/A/e4c4076a5e8e4364ae727617b7d236af, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/A/4c703a84861d4fc4acee2fad108ac3ad, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/A/d038446a3b4e463e8c833e29e20a8c22, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/A/1a5c886fee254cacb50b53c46d86168f, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/A/179955a798a84f0dae1b040d29ae9b26, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/A/6f7f946cbe4f444c9861a6d7e31488e3] to archive 2024-11-18T20:13:26,026 DEBUG [StoreCloser-TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-18T20:13:26,028 DEBUG [StoreCloser-TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/A/2f328b8a126d4d1a85c43d2f16bb4d46 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/A/2f328b8a126d4d1a85c43d2f16bb4d46 2024-11-18T20:13:26,028 DEBUG [StoreCloser-TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/A/68b7edf47fb44fad8c46066aa8579a85 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/A/68b7edf47fb44fad8c46066aa8579a85 2024-11-18T20:13:26,029 DEBUG [StoreCloser-TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/A/3cabbf51719548c8a5de6ec4f94917fc to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/A/3cabbf51719548c8a5de6ec4f94917fc 2024-11-18T20:13:26,030 DEBUG [StoreCloser-TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/A/88535c1b8c9e420994fb3e68ed8d74d9 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/A/88535c1b8c9e420994fb3e68ed8d74d9 2024-11-18T20:13:26,031 DEBUG [StoreCloser-TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/A/56e41c81598b46ac93b521ec008bc0fb to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/A/56e41c81598b46ac93b521ec008bc0fb 2024-11-18T20:13:26,031 DEBUG [StoreCloser-TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/A/e4181afa412d49b98b17bbf3a2c56f2a to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/A/e4181afa412d49b98b17bbf3a2c56f2a 2024-11-18T20:13:26,032 DEBUG [StoreCloser-TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/A/f0a1d5aafb0840f5a1fad13c65d79fbf to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/A/f0a1d5aafb0840f5a1fad13c65d79fbf 2024-11-18T20:13:26,033 DEBUG [StoreCloser-TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/A/3f0423b700404204bee394167d8140ce to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/A/3f0423b700404204bee394167d8140ce 2024-11-18T20:13:26,033 DEBUG [StoreCloser-TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/A/278c8263546e4e03a8190fffe4c44dde to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/A/278c8263546e4e03a8190fffe4c44dde 2024-11-18T20:13:26,034 DEBUG [StoreCloser-TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/A/f62e78b53c944147aa95cb47c8f64e96 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/A/f62e78b53c944147aa95cb47c8f64e96 2024-11-18T20:13:26,035 DEBUG [StoreCloser-TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/A/e53dd6b2568444f391f1c1726107c192 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/A/e53dd6b2568444f391f1c1726107c192 2024-11-18T20:13:26,035 DEBUG [StoreCloser-TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/A/c40f358a3dac4deca6a5df11adbbef03 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/A/c40f358a3dac4deca6a5df11adbbef03 2024-11-18T20:13:26,036 DEBUG [StoreCloser-TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/A/57592715ce4b460db24a60d036c13459 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/A/57592715ce4b460db24a60d036c13459 2024-11-18T20:13:26,037 DEBUG [StoreCloser-TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/A/2ca7ca5abfab41c49390187908b97e7b to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/A/2ca7ca5abfab41c49390187908b97e7b 2024-11-18T20:13:26,038 DEBUG [StoreCloser-TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/A/1a2f88f8f9b94234b63337e5b7a1da79 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/A/1a2f88f8f9b94234b63337e5b7a1da79 2024-11-18T20:13:26,039 DEBUG [StoreCloser-TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/A/f336b89ef6f440c5ae78966493d3e3ec to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/A/f336b89ef6f440c5ae78966493d3e3ec 2024-11-18T20:13:26,040 DEBUG [StoreCloser-TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/A/6bb29e1da64e40c0845d3a90926ee4ca to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/A/6bb29e1da64e40c0845d3a90926ee4ca 2024-11-18T20:13:26,041 DEBUG [StoreCloser-TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/A/747c2a99a3c446768093ceb49dce01d3 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/A/747c2a99a3c446768093ceb49dce01d3 2024-11-18T20:13:26,042 DEBUG [StoreCloser-TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/A/c292d643f1b9494d97c748f09bb89345 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/A/c292d643f1b9494d97c748f09bb89345 2024-11-18T20:13:26,043 DEBUG [StoreCloser-TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/A/9c4dfcac8a4e4a1c91db376f25e1744f to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/A/9c4dfcac8a4e4a1c91db376f25e1744f 2024-11-18T20:13:26,044 DEBUG [StoreCloser-TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/A/7fa7a394f3024177987167f47961229f to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/A/7fa7a394f3024177987167f47961229f 2024-11-18T20:13:26,045 DEBUG [StoreCloser-TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/A/4e7248aaa27746e885ad8d2a9fee9d72 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/A/4e7248aaa27746e885ad8d2a9fee9d72 2024-11-18T20:13:26,046 DEBUG [StoreCloser-TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/A/af73e07d2a5e4cdc9ec2ad846af20659 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/A/af73e07d2a5e4cdc9ec2ad846af20659 2024-11-18T20:13:26,047 DEBUG [StoreCloser-TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/A/d0938223c5274d37af2a9768626ce5a0 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/A/d0938223c5274d37af2a9768626ce5a0 2024-11-18T20:13:26,048 DEBUG [StoreCloser-TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/A/021c6f5e56764333892e460b10b4634f to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/A/021c6f5e56764333892e460b10b4634f 2024-11-18T20:13:26,049 DEBUG [StoreCloser-TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/A/e4c4076a5e8e4364ae727617b7d236af to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/A/e4c4076a5e8e4364ae727617b7d236af 2024-11-18T20:13:26,050 DEBUG [StoreCloser-TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/A/4c703a84861d4fc4acee2fad108ac3ad to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/A/4c703a84861d4fc4acee2fad108ac3ad 2024-11-18T20:13:26,051 DEBUG [StoreCloser-TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/A/d038446a3b4e463e8c833e29e20a8c22 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/A/d038446a3b4e463e8c833e29e20a8c22 2024-11-18T20:13:26,053 DEBUG [StoreCloser-TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/A/1a5c886fee254cacb50b53c46d86168f to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/A/1a5c886fee254cacb50b53c46d86168f 2024-11-18T20:13:26,054 DEBUG [StoreCloser-TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/A/179955a798a84f0dae1b040d29ae9b26 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/A/179955a798a84f0dae1b040d29ae9b26 2024-11-18T20:13:26,056 DEBUG [StoreCloser-TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/A/6f7f946cbe4f444c9861a6d7e31488e3 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/A/6f7f946cbe4f444c9861a6d7e31488e3 2024-11-18T20:13:26,058 DEBUG [StoreCloser-TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/B/02051258119b47c485c2b156ef4fe198, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/B/1356222be6be4ad089116c568b356d97, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/B/da5441a44a1c42ee96a4403bdbff8054, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/B/87bc89a38c1148d2859fd397687f49ae, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/B/a5f721f5cb0846dab70ce67344a0cbb1, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/B/2099464fb3d9496db4e841d557d7a196, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/B/5f5c0adbe79d490bb9b0beef5b618308, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/B/30488c808ef74e198607ddf904974164, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/B/9f3850dd8b8f44c896af45ecfb34d569, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/B/65accc17d5364351a4b5440e366bcbc7, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/B/ecfb2b52d5d94baeb8684fdb29c9fa8c, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/B/3068410e267748008678ea24f73ddc40, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/B/c79f73571fb04b528088d8034c5153e1, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/B/2f9e132a392a4c159750eeda0a04a4e1, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/B/20466a25c4c74ded8a242b3da0aaa76a, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/B/b57ed696696f47e5abb3f9cc7cb11b30, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/B/aa4a5d754cfc4a819020fe51384a849f, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/B/6ba2a1c8952147d4be8f837de5c4f49b, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/B/463dcbcaead9413a88c7640e95132d29, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/B/0ee038e7421a48c5898b8c84d946f0a6, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/B/1b167d039f734ca5bc88725e2fb064c2, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/B/a3cc67c317944dad81a6c4637b0bb49e, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/B/ffe8eb5fb24d447b8879e8f869a754df, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/B/b67a4da182ba43058d11d8a7d8b1ab4c, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/B/0bcb9c97a66a432797639ad418e14c0d, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/B/0e309084e07e42cfbbb801c25cd9b6f4, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/B/17ce2b3d10444368b72559b9ca9c6747, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/B/1b093495e501499285c4d3b10478b24a, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/B/264fb1f7d1b342eb850231cb31a51abc, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/B/e99e524319614518b88617e4e8dd15c6, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/B/37effb6494b54275a2748ec7f1e9b511] to archive 2024-11-18T20:13:26,059 DEBUG [StoreCloser-TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-18T20:13:26,061 DEBUG [StoreCloser-TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/B/02051258119b47c485c2b156ef4fe198 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/B/02051258119b47c485c2b156ef4fe198 2024-11-18T20:13:26,062 DEBUG [StoreCloser-TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/B/1356222be6be4ad089116c568b356d97 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/B/1356222be6be4ad089116c568b356d97 2024-11-18T20:13:26,063 DEBUG [StoreCloser-TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/B/da5441a44a1c42ee96a4403bdbff8054 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/B/da5441a44a1c42ee96a4403bdbff8054 2024-11-18T20:13:26,063 DEBUG [StoreCloser-TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/B/87bc89a38c1148d2859fd397687f49ae to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/B/87bc89a38c1148d2859fd397687f49ae 2024-11-18T20:13:26,064 DEBUG [StoreCloser-TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/B/a5f721f5cb0846dab70ce67344a0cbb1 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/B/a5f721f5cb0846dab70ce67344a0cbb1 2024-11-18T20:13:26,065 DEBUG [StoreCloser-TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/B/2099464fb3d9496db4e841d557d7a196 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/B/2099464fb3d9496db4e841d557d7a196 2024-11-18T20:13:26,066 DEBUG [StoreCloser-TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/B/5f5c0adbe79d490bb9b0beef5b618308 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/B/5f5c0adbe79d490bb9b0beef5b618308 2024-11-18T20:13:26,067 DEBUG [StoreCloser-TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/B/30488c808ef74e198607ddf904974164 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/B/30488c808ef74e198607ddf904974164 2024-11-18T20:13:26,067 DEBUG [StoreCloser-TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/B/9f3850dd8b8f44c896af45ecfb34d569 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/B/9f3850dd8b8f44c896af45ecfb34d569 2024-11-18T20:13:26,068 DEBUG [StoreCloser-TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/B/65accc17d5364351a4b5440e366bcbc7 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/B/65accc17d5364351a4b5440e366bcbc7 2024-11-18T20:13:26,069 DEBUG [StoreCloser-TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/B/ecfb2b52d5d94baeb8684fdb29c9fa8c to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/B/ecfb2b52d5d94baeb8684fdb29c9fa8c 2024-11-18T20:13:26,070 DEBUG [StoreCloser-TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/B/3068410e267748008678ea24f73ddc40 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/B/3068410e267748008678ea24f73ddc40 2024-11-18T20:13:26,071 DEBUG [StoreCloser-TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/B/c79f73571fb04b528088d8034c5153e1 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/B/c79f73571fb04b528088d8034c5153e1 2024-11-18T20:13:26,071 DEBUG [StoreCloser-TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/B/2f9e132a392a4c159750eeda0a04a4e1 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/B/2f9e132a392a4c159750eeda0a04a4e1 2024-11-18T20:13:26,072 DEBUG [StoreCloser-TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/B/20466a25c4c74ded8a242b3da0aaa76a to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/B/20466a25c4c74ded8a242b3da0aaa76a 2024-11-18T20:13:26,073 DEBUG [StoreCloser-TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/B/b57ed696696f47e5abb3f9cc7cb11b30 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/B/b57ed696696f47e5abb3f9cc7cb11b30 2024-11-18T20:13:26,074 DEBUG [StoreCloser-TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/B/aa4a5d754cfc4a819020fe51384a849f to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/B/aa4a5d754cfc4a819020fe51384a849f 2024-11-18T20:13:26,075 DEBUG [StoreCloser-TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/B/6ba2a1c8952147d4be8f837de5c4f49b to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/B/6ba2a1c8952147d4be8f837de5c4f49b 2024-11-18T20:13:26,075 DEBUG [StoreCloser-TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/B/463dcbcaead9413a88c7640e95132d29 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/B/463dcbcaead9413a88c7640e95132d29 2024-11-18T20:13:26,076 DEBUG [StoreCloser-TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/B/0ee038e7421a48c5898b8c84d946f0a6 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/B/0ee038e7421a48c5898b8c84d946f0a6 2024-11-18T20:13:26,077 DEBUG [StoreCloser-TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/B/1b167d039f734ca5bc88725e2fb064c2 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/B/1b167d039f734ca5bc88725e2fb064c2 2024-11-18T20:13:26,078 DEBUG [StoreCloser-TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/B/a3cc67c317944dad81a6c4637b0bb49e to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/B/a3cc67c317944dad81a6c4637b0bb49e 2024-11-18T20:13:26,079 DEBUG [StoreCloser-TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/B/ffe8eb5fb24d447b8879e8f869a754df to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/B/ffe8eb5fb24d447b8879e8f869a754df 2024-11-18T20:13:26,079 DEBUG [StoreCloser-TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/B/b67a4da182ba43058d11d8a7d8b1ab4c to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/B/b67a4da182ba43058d11d8a7d8b1ab4c 2024-11-18T20:13:26,080 DEBUG [StoreCloser-TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/B/0bcb9c97a66a432797639ad418e14c0d to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/B/0bcb9c97a66a432797639ad418e14c0d 2024-11-18T20:13:26,081 DEBUG [StoreCloser-TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/B/0e309084e07e42cfbbb801c25cd9b6f4 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/B/0e309084e07e42cfbbb801c25cd9b6f4 2024-11-18T20:13:26,082 DEBUG [StoreCloser-TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/B/17ce2b3d10444368b72559b9ca9c6747 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/B/17ce2b3d10444368b72559b9ca9c6747 2024-11-18T20:13:26,082 DEBUG [StoreCloser-TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/B/1b093495e501499285c4d3b10478b24a to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/B/1b093495e501499285c4d3b10478b24a 2024-11-18T20:13:26,083 DEBUG [StoreCloser-TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/B/264fb1f7d1b342eb850231cb31a51abc to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/B/264fb1f7d1b342eb850231cb31a51abc 2024-11-18T20:13:26,084 DEBUG [StoreCloser-TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/B/e99e524319614518b88617e4e8dd15c6 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/B/e99e524319614518b88617e4e8dd15c6 2024-11-18T20:13:26,085 DEBUG [StoreCloser-TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/B/37effb6494b54275a2748ec7f1e9b511 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/B/37effb6494b54275a2748ec7f1e9b511 2024-11-18T20:13:26,086 DEBUG [StoreCloser-TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/C/471fbc201cb14abba98e4f22c3276257, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/C/8060693bb295482fb8078d089a7d8fe8, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/C/2ea265c0be284e1c98b7bfd0922ee4ed, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/C/33d7ec8003794e2a9c97e3019a47912d, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/C/1ee05268c255479597687f7a1957da7a, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/C/a48911d413104538b1b658433f9a7afe, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/C/b360175e4f6448bd8e5deb4e8388de99, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/C/a6575a8c89cf4439bec4adfb0f820bb4, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/C/68e856d8a761480fbb963bb5f684ac45, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/C/2e557bf37f304aa5a2c3070e9c42ded9, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/C/bfbd2f8a966b4b348cc16e23b393b87d, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/C/028b665e5f3a4b3082348a8dd2ab2b6b, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/C/33b932f406b245c18ee7796a32cfbb45, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/C/5de0ea295e084184b934a2060fd62e1e, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/C/61247116172143a5a44b686ab32303af, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/C/47ea7d61bbb04e9ea44481e26914845a, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/C/72b31fc27c0b4a30ab629298c92b8c50, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/C/a8114ff7c0a340bb9c8248907401d3f2, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/C/5bcb2e7a514845e1a7115244e78fb5d0, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/C/d8d24908329c4a24bb0be3fc4dd78562, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/C/176916338d184fbcbc1d29112054c92e, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/C/6d36c0ed0f0e416681d6b49cc458d365, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/C/4f1afef087b747c186a2e24869c30aae, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/C/392a0ab7d9dd41c69c2bc6315bb86699, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/C/b525a90460c443438dc7cfbb263f1740, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/C/ae10870a09014e12a2f809b9199e133b, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/C/5c93d491f68d441586bc4655ea82b564, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/C/f8c18faca8634429bf036470865845b7, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/C/ad5b9ef5a15e40339947529315a8b274, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/C/a30540ccd8fc47eaa4191214a6901e31, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/C/b4c510965eb2467d96a4cc74ca3d409b] to archive 2024-11-18T20:13:26,087 DEBUG [StoreCloser-TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-18T20:13:26,088 DEBUG [StoreCloser-TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/C/471fbc201cb14abba98e4f22c3276257 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/C/471fbc201cb14abba98e4f22c3276257 2024-11-18T20:13:26,089 DEBUG [StoreCloser-TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/C/8060693bb295482fb8078d089a7d8fe8 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/C/8060693bb295482fb8078d089a7d8fe8 2024-11-18T20:13:26,090 DEBUG [StoreCloser-TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/C/2ea265c0be284e1c98b7bfd0922ee4ed to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/C/2ea265c0be284e1c98b7bfd0922ee4ed 2024-11-18T20:13:26,090 DEBUG [StoreCloser-TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/C/33d7ec8003794e2a9c97e3019a47912d to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/C/33d7ec8003794e2a9c97e3019a47912d 2024-11-18T20:13:26,091 DEBUG [StoreCloser-TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/C/1ee05268c255479597687f7a1957da7a to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/C/1ee05268c255479597687f7a1957da7a 2024-11-18T20:13:26,092 DEBUG [StoreCloser-TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/C/a48911d413104538b1b658433f9a7afe to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/C/a48911d413104538b1b658433f9a7afe 2024-11-18T20:13:26,093 DEBUG [StoreCloser-TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/C/b360175e4f6448bd8e5deb4e8388de99 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/C/b360175e4f6448bd8e5deb4e8388de99 2024-11-18T20:13:26,094 DEBUG [StoreCloser-TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/C/a6575a8c89cf4439bec4adfb0f820bb4 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/C/a6575a8c89cf4439bec4adfb0f820bb4 2024-11-18T20:13:26,094 DEBUG [StoreCloser-TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/C/68e856d8a761480fbb963bb5f684ac45 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/C/68e856d8a761480fbb963bb5f684ac45 2024-11-18T20:13:26,095 DEBUG [StoreCloser-TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/C/2e557bf37f304aa5a2c3070e9c42ded9 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/C/2e557bf37f304aa5a2c3070e9c42ded9 2024-11-18T20:13:26,096 DEBUG [StoreCloser-TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/C/bfbd2f8a966b4b348cc16e23b393b87d to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/C/bfbd2f8a966b4b348cc16e23b393b87d 2024-11-18T20:13:26,096 DEBUG [StoreCloser-TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/C/028b665e5f3a4b3082348a8dd2ab2b6b to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/C/028b665e5f3a4b3082348a8dd2ab2b6b 2024-11-18T20:13:26,097 DEBUG [StoreCloser-TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/C/33b932f406b245c18ee7796a32cfbb45 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/C/33b932f406b245c18ee7796a32cfbb45 2024-11-18T20:13:26,098 DEBUG [StoreCloser-TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/C/5de0ea295e084184b934a2060fd62e1e to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/C/5de0ea295e084184b934a2060fd62e1e 2024-11-18T20:13:26,099 DEBUG [StoreCloser-TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/C/61247116172143a5a44b686ab32303af to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/C/61247116172143a5a44b686ab32303af 2024-11-18T20:13:26,099 DEBUG [StoreCloser-TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/C/47ea7d61bbb04e9ea44481e26914845a to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/C/47ea7d61bbb04e9ea44481e26914845a 2024-11-18T20:13:26,100 DEBUG [StoreCloser-TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/C/72b31fc27c0b4a30ab629298c92b8c50 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/C/72b31fc27c0b4a30ab629298c92b8c50 2024-11-18T20:13:26,101 DEBUG [StoreCloser-TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/C/a8114ff7c0a340bb9c8248907401d3f2 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/C/a8114ff7c0a340bb9c8248907401d3f2 2024-11-18T20:13:26,101 DEBUG [StoreCloser-TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/C/5bcb2e7a514845e1a7115244e78fb5d0 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/C/5bcb2e7a514845e1a7115244e78fb5d0 2024-11-18T20:13:26,102 DEBUG [StoreCloser-TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/C/d8d24908329c4a24bb0be3fc4dd78562 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/C/d8d24908329c4a24bb0be3fc4dd78562 2024-11-18T20:13:26,103 DEBUG [StoreCloser-TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/C/176916338d184fbcbc1d29112054c92e to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/C/176916338d184fbcbc1d29112054c92e 2024-11-18T20:13:26,103 DEBUG [StoreCloser-TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/C/6d36c0ed0f0e416681d6b49cc458d365 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/C/6d36c0ed0f0e416681d6b49cc458d365 2024-11-18T20:13:26,104 DEBUG [StoreCloser-TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/C/4f1afef087b747c186a2e24869c30aae to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/C/4f1afef087b747c186a2e24869c30aae 2024-11-18T20:13:26,105 DEBUG [StoreCloser-TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/C/392a0ab7d9dd41c69c2bc6315bb86699 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/C/392a0ab7d9dd41c69c2bc6315bb86699 2024-11-18T20:13:26,105 DEBUG [StoreCloser-TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/C/b525a90460c443438dc7cfbb263f1740 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/C/b525a90460c443438dc7cfbb263f1740 2024-11-18T20:13:26,106 DEBUG [StoreCloser-TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/C/ae10870a09014e12a2f809b9199e133b to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/C/ae10870a09014e12a2f809b9199e133b 2024-11-18T20:13:26,106 DEBUG [StoreCloser-TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/C/5c93d491f68d441586bc4655ea82b564 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/C/5c93d491f68d441586bc4655ea82b564 2024-11-18T20:13:26,107 DEBUG [StoreCloser-TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/C/f8c18faca8634429bf036470865845b7 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/C/f8c18faca8634429bf036470865845b7 2024-11-18T20:13:26,108 DEBUG [StoreCloser-TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/C/ad5b9ef5a15e40339947529315a8b274 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/C/ad5b9ef5a15e40339947529315a8b274 2024-11-18T20:13:26,109 DEBUG [StoreCloser-TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/C/a30540ccd8fc47eaa4191214a6901e31 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/C/a30540ccd8fc47eaa4191214a6901e31 2024-11-18T20:13:26,109 DEBUG [StoreCloser-TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/C/b4c510965eb2467d96a4cc74ca3d409b to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/C/b4c510965eb2467d96a4cc74ca3d409b 2024-11-18T20:13:26,113 DEBUG [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/recovered.edits/475.seqid, newMaxSeqId=475, maxSeqId=1 2024-11-18T20:13:26,114 INFO [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70. 2024-11-18T20:13:26,114 DEBUG [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegion(1676): Region close journal for 294b8f086325976977089df95da38d70: Waiting for close lock at 1731960804756Running coprocessor pre-close hooks at 1731960804756Disabling compacts and flushes for region at 1731960804756Disabling writes for close at 1731960804756Obtaining lock to block concurrent updates at 1731960804757 (+1 ms)Preparing flush snapshotting stores in 294b8f086325976977089df95da38d70 at 1731960804757Finished memstore snapshotting TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=90720, getOffHeapSize=0, getCellsCount=750 at 1731960804757Flushing stores of TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70. at 1731960804758 (+1 ms)Flushing 294b8f086325976977089df95da38d70/A: creating writer at 1731960804758Flushing 294b8f086325976977089df95da38d70/A: appending metadata at 1731960804763 (+5 ms)Flushing 294b8f086325976977089df95da38d70/A: closing flushed file at 1731960804763Flushing 294b8f086325976977089df95da38d70/B: creating writer at 1731960805179 (+416 ms)Flushing 294b8f086325976977089df95da38d70/B: appending metadata at 1731960805181 (+2 ms)Flushing 294b8f086325976977089df95da38d70/B: closing flushed file at 1731960805182 (+1 ms)Flushing 294b8f086325976977089df95da38d70/C: creating writer at 1731960805594 (+412 ms)Flushing 294b8f086325976977089df95da38d70/C: appending metadata at 1731960805596 (+2 ms)Flushing 294b8f086325976977089df95da38d70/C: closing flushed file at 1731960805596Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@726660a9: reopening flushed file at 1731960806009 (+413 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2500104e: reopening flushed file at 1731960806016 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7610a04: reopening flushed file at 1731960806020 (+4 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for 294b8f086325976977089df95da38d70 in 1269ms, sequenceid=472, compaction requested=true at 1731960806025 (+5 ms)Writing region close event to WAL at 1731960806111 (+86 ms)Running coprocessor post-close hooks at 1731960806114 (+3 ms)Closed at 1731960806114 2024-11-18T20:13:26,115 INFO [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] handler.UnassignRegionHandler(157): Closed 294b8f086325976977089df95da38d70 2024-11-18T20:13:26,116 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=88 updating hbase:meta row=294b8f086325976977089df95da38d70, regionState=CLOSED 2024-11-18T20:13:26,117 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=89, ppid=88, state=RUNNABLE, hasLock=false; CloseRegionProcedure 294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 because future has completed 2024-11-18T20:13:26,119 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=89, resume processing ppid=88 2024-11-18T20:13:26,119 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=89, ppid=88, state=SUCCESS, hasLock=false; CloseRegionProcedure 294b8f086325976977089df95da38d70, server=c0a89b2656d4,35963,1731960718223 in 1.5150 sec 2024-11-18T20:13:26,120 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=88, resume processing ppid=87 2024-11-18T20:13:26,121 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=88, ppid=87, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=294b8f086325976977089df95da38d70, UNASSIGN in 1.5190 sec 2024-11-18T20:13:26,122 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=87, resume processing ppid=86 2024-11-18T20:13:26,122 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=87, ppid=86, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.5240 sec 2024-11-18T20:13:26,123 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731960806123"}]},"ts":"1731960806123"} 2024-11-18T20:13:26,124 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-18T20:13:26,124 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-18T20:13:26,125 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=86, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.5370 sec 2024-11-18T20:13:26,534 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-18T20:13:26,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-18T20:13:26,743 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-18T20:13:26,744 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-18T20:13:26,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] procedure2.ProcedureExecutor(1139): Stored pid=90, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-18T20:13:26,745 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=90, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-18T20:13:26,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=90 2024-11-18T20:13:26,746 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=90, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-18T20:13:26,748 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70 2024-11-18T20:13:26,750 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/A, FileablePath, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/B, FileablePath, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/C, FileablePath, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/recovered.edits] 2024-11-18T20:13:26,753 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/A/3164aceb482841ee82585018cc73eff1 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/A/3164aceb482841ee82585018cc73eff1 2024-11-18T20:13:26,754 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/A/9d9f10351eb24831a61ed819801929ed to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/A/9d9f10351eb24831a61ed819801929ed 2024-11-18T20:13:26,755 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/A/eb47b3e30a6d493b95cb7d84a36a7bef to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/A/eb47b3e30a6d493b95cb7d84a36a7bef 2024-11-18T20:13:26,756 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/B/3b2d15d05d104c9cb0e2603c0b68496a to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/B/3b2d15d05d104c9cb0e2603c0b68496a 2024-11-18T20:13:26,757 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/B/640c6b837e1b4b87adfa28586a3bea5b to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/B/640c6b837e1b4b87adfa28586a3bea5b 2024-11-18T20:13:26,758 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/B/a3475f5601dd4ea6b623b6af15878a2d to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/B/a3475f5601dd4ea6b623b6af15878a2d 2024-11-18T20:13:26,760 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/C/146c7cd638ca4a09ba6a72226c880dfc to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/C/146c7cd638ca4a09ba6a72226c880dfc 2024-11-18T20:13:26,760 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/C/2b43689b248e492d8a333a09d88f6acf to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/C/2b43689b248e492d8a333a09d88f6acf 2024-11-18T20:13:26,761 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/C/caa23bb7c5c54cfba70fb6531012d520 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/C/caa23bb7c5c54cfba70fb6531012d520 2024-11-18T20:13:26,763 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/recovered.edits/475.seqid to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70/recovered.edits/475.seqid 2024-11-18T20:13:26,763 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/294b8f086325976977089df95da38d70 2024-11-18T20:13:26,763 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-18T20:13:26,765 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=90, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-18T20:13:26,768 WARN [PEWorker-1 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-18T20:13:26,770 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-18T20:13:26,771 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=90, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-18T20:13:26,771 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-18T20:13:26,771 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731960806771"}]},"ts":"9223372036854775807"} 2024-11-18T20:13:26,773 INFO [PEWorker-1 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-18T20:13:26,773 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 294b8f086325976977089df95da38d70, NAME => 'TestAcidGuarantees,,1731960781264.294b8f086325976977089df95da38d70.', STARTKEY => '', ENDKEY => ''}] 2024-11-18T20:13:26,773 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-18T20:13:26,773 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731960806773"}]},"ts":"9223372036854775807"} 2024-11-18T20:13:26,775 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-18T20:13:26,776 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(133): Finished pid=90, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-18T20:13:26,777 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=90, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 32 msec 2024-11-18T20:13:26,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=90 2024-11-18T20:13:26,853 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-18T20:13:26,853 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-18T20:13:26,861 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testGetAtomicity Thread=239 (was 237) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_246716058_22 at /127.0.0.1:54554 [Waiting for operation #229] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1376622929_22 at /127.0.0.1:43250 [Waiting for operation #400] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=453 (was 453), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=513 (was 478) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=2583 (was 2718) 2024-11-18T20:13:26,869 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testMobScanAtomicity Thread=239, OpenFileDescriptor=453, MaxFileDescriptor=1048576, SystemLoadAverage=513, ProcessCount=11, AvailableMemoryMB=2580 2024-11-18T20:13:26,870 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-18T20:13:26,870 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-18T20:13:26,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] procedure2.ProcedureExecutor(1139): Stored pid=91, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-18T20:13:26,872 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=91, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-18T20:13:26,872 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:26,873 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 91 2024-11-18T20:13:26,873 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=91, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-18T20:13:26,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-11-18T20:13:26,877 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742167_1343 (size=959) 2024-11-18T20:13:26,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-11-18T20:13:27,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-11-18T20:13:27,283 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 8eaa77b8f4c3de4047bf5415370077cd, NAME => 'TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123 2024-11-18T20:13:27,292 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742168_1344 (size=53) 2024-11-18T20:13:27,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-11-18T20:13:27,693 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T20:13:27,693 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 8eaa77b8f4c3de4047bf5415370077cd, disabling compactions & flushes 2024-11-18T20:13:27,693 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd. 2024-11-18T20:13:27,693 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd. 2024-11-18T20:13:27,693 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd. after waiting 0 ms 2024-11-18T20:13:27,693 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd. 2024-11-18T20:13:27,693 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd. 2024-11-18T20:13:27,693 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 8eaa77b8f4c3de4047bf5415370077cd: Waiting for close lock at 1731960807693Disabling compacts and flushes for region at 1731960807693Disabling writes for close at 1731960807693Writing region close event to WAL at 1731960807693Closed at 1731960807693 2024-11-18T20:13:27,695 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=91, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-18T20:13:27,695 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731960807695"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731960807695"}]},"ts":"1731960807695"} 2024-11-18T20:13:27,698 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-18T20:13:27,699 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=91, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-18T20:13:27,699 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731960807699"}]},"ts":"1731960807699"} 2024-11-18T20:13:27,701 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-18T20:13:27,701 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=92, ppid=91, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=8eaa77b8f4c3de4047bf5415370077cd, ASSIGN}] 2024-11-18T20:13:27,703 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=92, ppid=91, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=8eaa77b8f4c3de4047bf5415370077cd, ASSIGN 2024-11-18T20:13:27,704 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=92, ppid=91, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=8eaa77b8f4c3de4047bf5415370077cd, ASSIGN; state=OFFLINE, location=c0a89b2656d4,35963,1731960718223; forceNewPlan=false, retain=false 2024-11-18T20:13:27,855 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=92 updating hbase:meta row=8eaa77b8f4c3de4047bf5415370077cd, regionState=OPENING, regionLocation=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:27,859 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=92, ppid=91, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=8eaa77b8f4c3de4047bf5415370077cd, ASSIGN because future has completed 2024-11-18T20:13:27,860 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=93, ppid=92, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223}] 2024-11-18T20:13:27,922 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-18T20:13:28,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-11-18T20:13:28,022 INFO [RS_OPEN_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd. 2024-11-18T20:13:28,023 DEBUG [RS_OPEN_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.HRegion(7752): Opening region: {ENCODED => 8eaa77b8f4c3de4047bf5415370077cd, NAME => 'TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd.', STARTKEY => '', ENDKEY => ''} 2024-11-18T20:13:28,024 DEBUG [RS_OPEN_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 8eaa77b8f4c3de4047bf5415370077cd 2024-11-18T20:13:28,024 DEBUG [RS_OPEN_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T20:13:28,024 DEBUG [RS_OPEN_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.HRegion(7794): checking encryption for 8eaa77b8f4c3de4047bf5415370077cd 2024-11-18T20:13:28,024 DEBUG [RS_OPEN_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.HRegion(7797): checking classloading for 8eaa77b8f4c3de4047bf5415370077cd 2024-11-18T20:13:28,026 INFO [StoreOpener-8eaa77b8f4c3de4047bf5415370077cd-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 8eaa77b8f4c3de4047bf5415370077cd 2024-11-18T20:13:28,028 INFO [StoreOpener-8eaa77b8f4c3de4047bf5415370077cd-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8eaa77b8f4c3de4047bf5415370077cd columnFamilyName A 2024-11-18T20:13:28,028 DEBUG [StoreOpener-8eaa77b8f4c3de4047bf5415370077cd-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:28,029 INFO [StoreOpener-8eaa77b8f4c3de4047bf5415370077cd-1 {}] regionserver.HStore(327): Store=8eaa77b8f4c3de4047bf5415370077cd/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T20:13:28,029 INFO [StoreOpener-8eaa77b8f4c3de4047bf5415370077cd-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 8eaa77b8f4c3de4047bf5415370077cd 2024-11-18T20:13:28,030 INFO [StoreOpener-8eaa77b8f4c3de4047bf5415370077cd-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8eaa77b8f4c3de4047bf5415370077cd columnFamilyName B 2024-11-18T20:13:28,030 DEBUG [StoreOpener-8eaa77b8f4c3de4047bf5415370077cd-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:28,031 INFO [StoreOpener-8eaa77b8f4c3de4047bf5415370077cd-1 {}] regionserver.HStore(327): Store=8eaa77b8f4c3de4047bf5415370077cd/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T20:13:28,031 INFO [StoreOpener-8eaa77b8f4c3de4047bf5415370077cd-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 8eaa77b8f4c3de4047bf5415370077cd 2024-11-18T20:13:28,032 INFO [StoreOpener-8eaa77b8f4c3de4047bf5415370077cd-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8eaa77b8f4c3de4047bf5415370077cd columnFamilyName C 2024-11-18T20:13:28,032 DEBUG [StoreOpener-8eaa77b8f4c3de4047bf5415370077cd-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:28,033 INFO [StoreOpener-8eaa77b8f4c3de4047bf5415370077cd-1 {}] regionserver.HStore(327): Store=8eaa77b8f4c3de4047bf5415370077cd/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T20:13:28,033 DEBUG [RS_OPEN_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.HRegion(1038): replaying wal for 8eaa77b8f4c3de4047bf5415370077cd 2024-11-18T20:13:28,034 DEBUG [RS_OPEN_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd 2024-11-18T20:13:28,034 DEBUG [RS_OPEN_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd 2024-11-18T20:13:28,036 DEBUG [RS_OPEN_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.HRegion(1048): stopping wal replay for 8eaa77b8f4c3de4047bf5415370077cd 2024-11-18T20:13:28,036 DEBUG [RS_OPEN_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.HRegion(1060): Cleaning up temporary data for 8eaa77b8f4c3de4047bf5415370077cd 2024-11-18T20:13:28,036 DEBUG [RS_OPEN_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-18T20:13:28,038 DEBUG [RS_OPEN_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.HRegion(1093): writing seq id for 8eaa77b8f4c3de4047bf5415370077cd 2024-11-18T20:13:28,040 DEBUG [RS_OPEN_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-18T20:13:28,040 INFO [RS_OPEN_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.HRegion(1114): Opened 8eaa77b8f4c3de4047bf5415370077cd; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66973361, jitterRate=-0.002019152045249939}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-18T20:13:28,041 DEBUG [RS_OPEN_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 8eaa77b8f4c3de4047bf5415370077cd 2024-11-18T20:13:28,041 DEBUG [RS_OPEN_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.HRegion(1006): Region open journal for 8eaa77b8f4c3de4047bf5415370077cd: Running coprocessor pre-open hook at 1731960808025Writing region info on filesystem at 1731960808025Initializing all the Stores at 1731960808026 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731960808026Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731960808026Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731960808026Cleaning up temporary data from old regions at 1731960808036 (+10 ms)Running coprocessor post-open hooks at 1731960808041 (+5 ms)Region opened successfully at 1731960808041 2024-11-18T20:13:28,042 INFO [RS_OPEN_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., pid=93, masterSystemTime=1731960808015 2024-11-18T20:13:28,043 DEBUG [RS_OPEN_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd. 2024-11-18T20:13:28,043 INFO [RS_OPEN_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd. 2024-11-18T20:13:28,044 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=92 updating hbase:meta row=8eaa77b8f4c3de4047bf5415370077cd, regionState=OPEN, openSeqNum=2, regionLocation=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:28,045 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=93, ppid=92, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 because future has completed 2024-11-18T20:13:28,046 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=93, resume processing ppid=92 2024-11-18T20:13:28,046 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=93, ppid=92, state=SUCCESS, hasLock=false; OpenRegionProcedure 8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 in 185 msec 2024-11-18T20:13:28,048 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=92, resume processing ppid=91 2024-11-18T20:13:28,048 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=92, ppid=91, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=8eaa77b8f4c3de4047bf5415370077cd, ASSIGN in 345 msec 2024-11-18T20:13:28,048 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=91, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-18T20:13:28,049 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731960808048"}]},"ts":"1731960808048"} 2024-11-18T20:13:28,049 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-18T20:13:28,050 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=91, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-18T20:13:28,051 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=91, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1800 sec 2024-11-18T20:13:29,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-11-18T20:13:29,034 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-18T20:13:29,036 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1ed6dd65, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:13:29,036 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c0a89b2656d4,41277,-1 for getting cluster id 2024-11-18T20:13:29,036 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T20:13:29,038 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ac99df6b-7791-4693-a81a-6a5a39504490' 2024-11-18T20:13:29,038 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T20:13:29,038 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ac99df6b-7791-4693-a81a-6a5a39504490" 2024-11-18T20:13:29,039 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@330f77e8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:13:29,039 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c0a89b2656d4,41277,-1] 2024-11-18T20:13:29,039 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T20:13:29,040 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:13:29,041 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58802, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T20:13:29,042 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4fcc2ac0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:13:29,042 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T20:13:29,043 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c0a89b2656d4,35963,1731960718223, seqNum=-1] 2024-11-18T20:13:29,043 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T20:13:29,044 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39830, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T20:13:29,045 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-18T20:13:29,046 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is c0a89b2656d4,41277,1731960717534 2024-11-18T20:13:29,046 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@559c75d0 2024-11-18T20:13:29,046 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-18T20:13:29,047 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58818, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-18T20:13:29,048 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-18T20:13:29,049 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-18T20:13:29,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] procedure2.ProcedureExecutor(1139): Stored pid=94, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-18T20:13:29,059 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742169_1345 (size=995) 2024-11-18T20:13:29,462 DEBUG [PEWorker-3 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.959 2024-11-18T20:13:29,462 INFO [PEWorker-3 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.995 2024-11-18T20:13:29,465 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=95, ppid=94, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-18T20:13:29,468 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=96, ppid=95, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=8eaa77b8f4c3de4047bf5415370077cd, REOPEN/MOVE}] 2024-11-18T20:13:29,469 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=96, ppid=95, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=8eaa77b8f4c3de4047bf5415370077cd, REOPEN/MOVE 2024-11-18T20:13:29,470 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=96 updating hbase:meta row=8eaa77b8f4c3de4047bf5415370077cd, regionState=CLOSING, regionLocation=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:29,471 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=96, ppid=95, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=8eaa77b8f4c3de4047bf5415370077cd, REOPEN/MOVE because future has completed 2024-11-18T20:13:29,471 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-18T20:13:29,472 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=97, ppid=96, state=RUNNABLE, hasLock=false; CloseRegionProcedure 8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223}] 2024-11-18T20:13:29,626 INFO [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] handler.UnassignRegionHandler(122): Close 8eaa77b8f4c3de4047bf5415370077cd 2024-11-18T20:13:29,626 DEBUG [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-18T20:13:29,626 DEBUG [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] regionserver.HRegion(1722): Closing 8eaa77b8f4c3de4047bf5415370077cd, disabling compactions & flushes 2024-11-18T20:13:29,626 INFO [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd. 2024-11-18T20:13:29,626 DEBUG [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd. 2024-11-18T20:13:29,626 DEBUG [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd. after waiting 0 ms 2024-11-18T20:13:29,627 DEBUG [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd. 2024-11-18T20:13:29,634 DEBUG [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-18T20:13:29,635 INFO [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd. 2024-11-18T20:13:29,635 DEBUG [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] regionserver.HRegion(1676): Region close journal for 8eaa77b8f4c3de4047bf5415370077cd: Waiting for close lock at 1731960809626Running coprocessor pre-close hooks at 1731960809626Disabling compacts and flushes for region at 1731960809626Disabling writes for close at 1731960809627 (+1 ms)Writing region close event to WAL at 1731960809628 (+1 ms)Running coprocessor post-close hooks at 1731960809634 (+6 ms)Closed at 1731960809634 2024-11-18T20:13:29,635 WARN [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] regionserver.HRegionServer(3299): Not adding moved region record: 8eaa77b8f4c3de4047bf5415370077cd to self. 2024-11-18T20:13:29,637 INFO [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] handler.UnassignRegionHandler(157): Closed 8eaa77b8f4c3de4047bf5415370077cd 2024-11-18T20:13:29,638 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=96 updating hbase:meta row=8eaa77b8f4c3de4047bf5415370077cd, regionState=CLOSED 2024-11-18T20:13:29,640 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=97, ppid=96, state=RUNNABLE, hasLock=false; CloseRegionProcedure 8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 because future has completed 2024-11-18T20:13:29,643 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=97, resume processing ppid=96 2024-11-18T20:13:29,643 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=97, ppid=96, state=SUCCESS, hasLock=false; CloseRegionProcedure 8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 in 170 msec 2024-11-18T20:13:29,644 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=96, ppid=95, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=8eaa77b8f4c3de4047bf5415370077cd, REOPEN/MOVE; state=CLOSED, location=c0a89b2656d4,35963,1731960718223; forceNewPlan=false, retain=true 2024-11-18T20:13:29,795 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=96 updating hbase:meta row=8eaa77b8f4c3de4047bf5415370077cd, regionState=OPENING, regionLocation=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:29,797 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=96, ppid=95, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=8eaa77b8f4c3de4047bf5415370077cd, REOPEN/MOVE because future has completed 2024-11-18T20:13:29,798 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=98, ppid=96, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223}] 2024-11-18T20:13:29,956 INFO [RS_OPEN_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd. 2024-11-18T20:13:29,957 DEBUG [RS_OPEN_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.HRegion(7752): Opening region: {ENCODED => 8eaa77b8f4c3de4047bf5415370077cd, NAME => 'TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd.', STARTKEY => '', ENDKEY => ''} 2024-11-18T20:13:29,958 DEBUG [RS_OPEN_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 8eaa77b8f4c3de4047bf5415370077cd 2024-11-18T20:13:29,958 DEBUG [RS_OPEN_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T20:13:29,958 DEBUG [RS_OPEN_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.HRegion(7794): checking encryption for 8eaa77b8f4c3de4047bf5415370077cd 2024-11-18T20:13:29,958 DEBUG [RS_OPEN_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.HRegion(7797): checking classloading for 8eaa77b8f4c3de4047bf5415370077cd 2024-11-18T20:13:29,962 INFO [StoreOpener-8eaa77b8f4c3de4047bf5415370077cd-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 8eaa77b8f4c3de4047bf5415370077cd 2024-11-18T20:13:29,963 INFO [StoreOpener-8eaa77b8f4c3de4047bf5415370077cd-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8eaa77b8f4c3de4047bf5415370077cd columnFamilyName A 2024-11-18T20:13:29,964 DEBUG [StoreOpener-8eaa77b8f4c3de4047bf5415370077cd-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:29,964 INFO [StoreOpener-8eaa77b8f4c3de4047bf5415370077cd-1 {}] regionserver.HStore(327): Store=8eaa77b8f4c3de4047bf5415370077cd/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T20:13:29,965 INFO [StoreOpener-8eaa77b8f4c3de4047bf5415370077cd-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 8eaa77b8f4c3de4047bf5415370077cd 2024-11-18T20:13:29,965 INFO [StoreOpener-8eaa77b8f4c3de4047bf5415370077cd-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8eaa77b8f4c3de4047bf5415370077cd columnFamilyName B 2024-11-18T20:13:29,966 DEBUG [StoreOpener-8eaa77b8f4c3de4047bf5415370077cd-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:29,966 INFO [StoreOpener-8eaa77b8f4c3de4047bf5415370077cd-1 {}] regionserver.HStore(327): Store=8eaa77b8f4c3de4047bf5415370077cd/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T20:13:29,966 INFO [StoreOpener-8eaa77b8f4c3de4047bf5415370077cd-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 8eaa77b8f4c3de4047bf5415370077cd 2024-11-18T20:13:29,967 INFO [StoreOpener-8eaa77b8f4c3de4047bf5415370077cd-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8eaa77b8f4c3de4047bf5415370077cd columnFamilyName C 2024-11-18T20:13:29,967 DEBUG [StoreOpener-8eaa77b8f4c3de4047bf5415370077cd-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:29,967 INFO [StoreOpener-8eaa77b8f4c3de4047bf5415370077cd-1 {}] regionserver.HStore(327): Store=8eaa77b8f4c3de4047bf5415370077cd/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T20:13:29,967 DEBUG [RS_OPEN_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.HRegion(1038): replaying wal for 8eaa77b8f4c3de4047bf5415370077cd 2024-11-18T20:13:29,968 DEBUG [RS_OPEN_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd 2024-11-18T20:13:29,969 DEBUG [RS_OPEN_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd 2024-11-18T20:13:29,970 DEBUG [RS_OPEN_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.HRegion(1048): stopping wal replay for 8eaa77b8f4c3de4047bf5415370077cd 2024-11-18T20:13:29,970 DEBUG [RS_OPEN_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.HRegion(1060): Cleaning up temporary data for 8eaa77b8f4c3de4047bf5415370077cd 2024-11-18T20:13:29,970 DEBUG [RS_OPEN_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-18T20:13:29,971 DEBUG [RS_OPEN_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.HRegion(1093): writing seq id for 8eaa77b8f4c3de4047bf5415370077cd 2024-11-18T20:13:29,972 INFO [RS_OPEN_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.HRegion(1114): Opened 8eaa77b8f4c3de4047bf5415370077cd; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73762776, jitterRate=0.09915101528167725}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-18T20:13:29,972 DEBUG [RS_OPEN_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 8eaa77b8f4c3de4047bf5415370077cd 2024-11-18T20:13:29,972 DEBUG [RS_OPEN_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.HRegion(1006): Region open journal for 8eaa77b8f4c3de4047bf5415370077cd: Running coprocessor pre-open hook at 1731960809959Writing region info on filesystem at 1731960809959Initializing all the Stores at 1731960809961 (+2 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731960809961Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731960809961Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731960809962 (+1 ms)Cleaning up temporary data from old regions at 1731960809970 (+8 ms)Running coprocessor post-open hooks at 1731960809972 (+2 ms)Region opened successfully at 1731960809972 2024-11-18T20:13:29,973 INFO [RS_OPEN_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., pid=98, masterSystemTime=1731960809951 2024-11-18T20:13:29,974 DEBUG [RS_OPEN_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd. 2024-11-18T20:13:29,974 INFO [RS_OPEN_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd. 2024-11-18T20:13:29,975 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=96 updating hbase:meta row=8eaa77b8f4c3de4047bf5415370077cd, regionState=OPEN, openSeqNum=5, regionLocation=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:29,976 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=98, ppid=96, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 because future has completed 2024-11-18T20:13:29,978 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=98, resume processing ppid=96 2024-11-18T20:13:29,978 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=98, ppid=96, state=SUCCESS, hasLock=false; OpenRegionProcedure 8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 in 178 msec 2024-11-18T20:13:29,979 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=96, resume processing ppid=95 2024-11-18T20:13:29,979 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=96, ppid=95, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=8eaa77b8f4c3de4047bf5415370077cd, REOPEN/MOVE in 510 msec 2024-11-18T20:13:29,982 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=95, resume processing ppid=94 2024-11-18T20:13:29,982 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=95, ppid=94, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 515 msec 2024-11-18T20:13:29,985 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=94, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 934 msec 2024-11-18T20:13:29,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=94 2024-11-18T20:13:29,986 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-18T20:13:29,986 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5d3de22, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:13:29,986 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c0a89b2656d4,41277,-1 for getting cluster id 2024-11-18T20:13:29,987 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T20:13:29,987 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ac99df6b-7791-4693-a81a-6a5a39504490' 2024-11-18T20:13:29,988 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T20:13:29,988 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ac99df6b-7791-4693-a81a-6a5a39504490" 2024-11-18T20:13:29,988 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7f471a3a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:13:29,988 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c0a89b2656d4,41277,-1] 2024-11-18T20:13:29,988 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T20:13:29,988 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:13:29,989 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58834, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T20:13:29,989 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@37f4940c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:13:29,990 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1774ef6a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:13:29,990 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c0a89b2656d4,41277,-1 for getting cluster id 2024-11-18T20:13:29,990 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T20:13:29,990 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ac99df6b-7791-4693-a81a-6a5a39504490' 2024-11-18T20:13:29,991 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T20:13:29,991 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ac99df6b-7791-4693-a81a-6a5a39504490" 2024-11-18T20:13:29,991 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@f5725c9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:13:29,991 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c0a89b2656d4,41277,-1] 2024-11-18T20:13:29,991 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T20:13:29,991 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:13:29,992 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58856, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T20:13:29,992 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5a85764, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:13:29,993 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1cf7f63f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:13:29,993 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c0a89b2656d4,41277,-1 for getting cluster id 2024-11-18T20:13:29,993 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T20:13:29,993 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ac99df6b-7791-4693-a81a-6a5a39504490' 2024-11-18T20:13:29,993 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T20:13:29,994 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ac99df6b-7791-4693-a81a-6a5a39504490" 2024-11-18T20:13:29,994 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7966fe8f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:13:29,994 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c0a89b2656d4,41277,-1] 2024-11-18T20:13:29,994 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T20:13:29,994 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:13:29,995 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58876, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T20:13:29,995 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1ae42470, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:13:29,996 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4600746, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:13:29,996 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c0a89b2656d4,41277,-1 for getting cluster id 2024-11-18T20:13:29,996 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T20:13:29,996 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ac99df6b-7791-4693-a81a-6a5a39504490' 2024-11-18T20:13:29,996 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T20:13:29,996 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ac99df6b-7791-4693-a81a-6a5a39504490" 2024-11-18T20:13:29,996 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@22e07fa8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:13:29,996 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c0a89b2656d4,41277,-1] 2024-11-18T20:13:29,997 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T20:13:29,997 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:13:29,997 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58894, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T20:13:29,998 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2633aa4f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:13:29,998 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6447def3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:13:29,998 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c0a89b2656d4,41277,-1 for getting cluster id 2024-11-18T20:13:29,999 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T20:13:29,999 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ac99df6b-7791-4693-a81a-6a5a39504490' 2024-11-18T20:13:29,999 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T20:13:29,999 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ac99df6b-7791-4693-a81a-6a5a39504490" 2024-11-18T20:13:29,999 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@35a7de82, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:13:29,999 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c0a89b2656d4,41277,-1] 2024-11-18T20:13:30,000 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T20:13:30,000 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:13:30,000 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58904, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T20:13:30,001 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1ea6c302, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:13:30,001 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@72e71a40, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:13:30,001 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c0a89b2656d4,41277,-1 for getting cluster id 2024-11-18T20:13:30,001 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T20:13:30,002 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ac99df6b-7791-4693-a81a-6a5a39504490' 2024-11-18T20:13:30,002 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T20:13:30,002 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ac99df6b-7791-4693-a81a-6a5a39504490" 2024-11-18T20:13:30,002 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2c02578e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:13:30,002 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c0a89b2656d4,41277,-1] 2024-11-18T20:13:30,002 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T20:13:30,002 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:13:30,003 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58918, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T20:13:30,003 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@201a8cca, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:13:30,004 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7b9dbe2c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:13:30,004 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c0a89b2656d4,41277,-1 for getting cluster id 2024-11-18T20:13:30,004 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T20:13:30,004 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ac99df6b-7791-4693-a81a-6a5a39504490' 2024-11-18T20:13:30,005 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T20:13:30,005 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ac99df6b-7791-4693-a81a-6a5a39504490" 2024-11-18T20:13:30,005 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4336cc5b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:13:30,005 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c0a89b2656d4,41277,-1] 2024-11-18T20:13:30,005 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T20:13:30,005 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:13:30,006 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58932, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T20:13:30,006 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@78ed923b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:13:30,006 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7eba3e2f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:13:30,006 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c0a89b2656d4,41277,-1 for getting cluster id 2024-11-18T20:13:30,007 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T20:13:30,007 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ac99df6b-7791-4693-a81a-6a5a39504490' 2024-11-18T20:13:30,007 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T20:13:30,007 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ac99df6b-7791-4693-a81a-6a5a39504490" 2024-11-18T20:13:30,007 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@78fce680, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:13:30,007 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c0a89b2656d4,41277,-1] 2024-11-18T20:13:30,007 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T20:13:30,008 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:13:30,008 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58936, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T20:13:30,008 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@f919ce6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:13:30,009 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@79570776, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:13:30,009 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c0a89b2656d4,41277,-1 for getting cluster id 2024-11-18T20:13:30,009 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T20:13:30,010 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ac99df6b-7791-4693-a81a-6a5a39504490' 2024-11-18T20:13:30,010 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T20:13:30,010 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ac99df6b-7791-4693-a81a-6a5a39504490" 2024-11-18T20:13:30,010 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@54abb83d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:13:30,010 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c0a89b2656d4,41277,-1] 2024-11-18T20:13:30,010 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T20:13:30,010 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:13:30,011 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58960, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T20:13:30,011 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@34ec8f44, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:13:30,012 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@43065a6b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:13:30,012 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c0a89b2656d4,41277,-1 for getting cluster id 2024-11-18T20:13:30,012 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T20:13:30,012 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ac99df6b-7791-4693-a81a-6a5a39504490' 2024-11-18T20:13:30,012 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T20:13:30,012 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ac99df6b-7791-4693-a81a-6a5a39504490" 2024-11-18T20:13:30,013 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3df5e210, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:13:30,013 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c0a89b2656d4,41277,-1] 2024-11-18T20:13:30,013 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T20:13:30,013 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:13:30,014 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58980, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T20:13:30,014 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4d5e6f1e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:13:30,015 DEBUG [Thread-1567 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T20:13:30,015 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c0a89b2656d4,35963,1731960718223, seqNum=-1] 2024-11-18T20:13:30,015 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T20:13:30,016 DEBUG [Thread-1580 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T20:13:30,016 DEBUG [Thread-1582 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T20:13:30,016 DEBUG [Thread-1586 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T20:13:30,016 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39834, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T20:13:30,017 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c0a89b2656d4,35963,1731960718223, seqNum=-1] 2024-11-18T20:13:30,017 DEBUG [Thread-1569 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T20:13:30,017 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c0a89b2656d4,35963,1731960718223, seqNum=-1] 2024-11-18T20:13:30,017 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T20:13:30,017 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c0a89b2656d4,35963,1731960718223, seqNum=-1] 2024-11-18T20:13:30,017 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T20:13:30,017 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T20:13:30,018 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c0a89b2656d4,35963,1731960718223, seqNum=-1] 2024-11-18T20:13:30,018 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39846, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T20:13:30,018 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39848, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T20:13:30,018 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T20:13:30,018 DEBUG [Thread-1584 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T20:13:30,018 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39860, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T20:13:30,019 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c0a89b2656d4,35963,1731960718223, seqNum=-1] 2024-11-18T20:13:30,019 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39866, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T20:13:30,019 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T20:13:30,020 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5] 2024-11-18T20:13:30,020 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5] 2024-11-18T20:13:30,020 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39880, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T20:13:30,021 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5] 2024-11-18T20:13:30,021 DEBUG [Thread-1575 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T20:13:30,022 DEBUG [Thread-1578 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T20:13:30,022 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41277 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-18T20:13:30,022 DEBUG [Thread-1571 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T20:13:30,022 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5] 2024-11-18T20:13:30,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41277 {}] procedure2.ProcedureExecutor(1139): Stored pid=99, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees 2024-11-18T20:13:30,023 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5] 2024-11-18T20:13:30,023 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5] 2024-11-18T20:13:30,023 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c0a89b2656d4,35963,1731960718223, seqNum=-1] 2024-11-18T20:13:30,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-18T20:13:30,023 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c0a89b2656d4,35963,1731960718223, seqNum=-1] 2024-11-18T20:13:30,024 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T20:13:30,024 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T20:13:30,024 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=99, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-18T20:13:30,024 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39882, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T20:13:30,025 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c0a89b2656d4,35963,1731960718223, seqNum=-1] 2024-11-18T20:13:30,025 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39896, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T20:13:30,025 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T20:13:30,025 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=99, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-18T20:13:30,026 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=100, ppid=99, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-18T20:13:30,026 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39910, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T20:13:30,027 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5] 2024-11-18T20:13:30,027 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5] 2024-11-18T20:13:30,029 DEBUG [Thread-1573 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T20:13:30,029 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5] 2024-11-18T20:13:30,030 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c0a89b2656d4,35963,1731960718223, seqNum=-1] 2024-11-18T20:13:30,030 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T20:13:30,035 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39924, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T20:13:30,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(8855): Flush requested on 8eaa77b8f4c3de4047bf5415370077cd 2024-11-18T20:13:30,039 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-18T20:13:30,039 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8eaa77b8f4c3de4047bf5415370077cd 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-18T20:13:30,041 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5] 2024-11-18T20:13:30,074 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118b082c61d9f9e49739f43f197b942b9a5_8eaa77b8f4c3de4047bf5415370077cd is 50, key is test_row_0/A:col10/1731960810036/Put/seqid=0 2024-11-18T20:13:30,088 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742170_1346 (size=14594) 2024-11-18T20:13:30,096 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:30,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39924 deadline: 1731960870087, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:30,096 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:30,096 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:30,096 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:30,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39882 deadline: 1731960870090, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:30,096 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:13:30,096 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:30,096 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:30,096 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:13:30,102 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:30,102 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:30,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39910 deadline: 1731960870093, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:30,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39866 deadline: 1731960870091, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:30,102 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:30,102 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:30,102 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:30,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39834 deadline: 1731960870097, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:30,102 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:13:30,102 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:30,103 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:30,103 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:13:30,104 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:30,104 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:30,104 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:13:30,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-18T20:13:30,181 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35963 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-18T20:13:30,181 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd. 2024-11-18T20:13:30,181 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd. as already flushing 2024-11-18T20:13:30,181 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd. 2024-11-18T20:13:30,181 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 8eaa77b8f4c3de4047bf5415370077cd, NAME => 'TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:13:30,181 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 8eaa77b8f4c3de4047bf5415370077cd, NAME => 'TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:13:30,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8eaa77b8f4c3de4047bf5415370077cd, NAME => 'TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8eaa77b8f4c3de4047bf5415370077cd, NAME => 'TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:13:30,209 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:30,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39924 deadline: 1731960870203, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:30,210 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:30,210 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:30,210 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:30,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39882 deadline: 1731960870203, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:30,210 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:13:30,210 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:30,210 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:30,210 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:13:30,219 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:30,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39866 deadline: 1731960870213, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:30,219 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:30,219 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:30,219 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:30,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39834 deadline: 1731960870213, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:30,219 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:13:30,220 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:30,220 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:30,220 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:30,220 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:13:30,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39910 deadline: 1731960870213, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:30,220 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:30,220 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:30,220 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:13:30,334 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35963 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-18T20:13:30,334 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd. 2024-11-18T20:13:30,334 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd. as already flushing 2024-11-18T20:13:30,334 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd. 2024-11-18T20:13:30,334 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 8eaa77b8f4c3de4047bf5415370077cd, NAME => 'TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:13:30,334 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 8eaa77b8f4c3de4047bf5415370077cd, NAME => 'TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:13:30,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8eaa77b8f4c3de4047bf5415370077cd, NAME => 'TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8eaa77b8f4c3de4047bf5415370077cd, NAME => 'TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:13:30,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-18T20:13:30,418 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:30,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39924 deadline: 1731960870413, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:30,418 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:30,418 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:30,418 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:30,418 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:13:30,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39882 deadline: 1731960870413, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:30,418 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:30,418 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:30,418 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:13:30,426 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:30,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39866 deadline: 1731960870422, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:30,426 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:30,426 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:30,426 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:30,426 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:13:30,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39834 deadline: 1731960870423, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:30,426 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:30,426 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:30,426 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:13:30,426 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:30,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39910 deadline: 1731960870423, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:30,427 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:30,427 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:30,427 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:13:30,486 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35963 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-18T20:13:30,486 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd. 2024-11-18T20:13:30,486 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd. as already flushing 2024-11-18T20:13:30,486 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd. 2024-11-18T20:13:30,486 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 8eaa77b8f4c3de4047bf5415370077cd, NAME => 'TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:13:30,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 8eaa77b8f4c3de4047bf5415370077cd, NAME => 'TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:13:30,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8eaa77b8f4c3de4047bf5415370077cd, NAME => 'TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8eaa77b8f4c3de4047bf5415370077cd, NAME => 'TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:13:30,489 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:30,492 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118b082c61d9f9e49739f43f197b942b9a5_8eaa77b8f4c3de4047bf5415370077cd to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118b082c61d9f9e49739f43f197b942b9a5_8eaa77b8f4c3de4047bf5415370077cd 2024-11-18T20:13:30,493 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/.tmp/A/13725551bd2245be8850dcc8a4559260, store: [table=TestAcidGuarantees family=A region=8eaa77b8f4c3de4047bf5415370077cd] 2024-11-18T20:13:30,493 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/.tmp/A/13725551bd2245be8850dcc8a4559260 is 175, key is test_row_0/A:col10/1731960810036/Put/seqid=0 2024-11-18T20:13:30,496 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742171_1347 (size=39549) 2024-11-18T20:13:30,498 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=16, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/.tmp/A/13725551bd2245be8850dcc8a4559260 2024-11-18T20:13:30,517 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/.tmp/B/cbd2cfe931ca46bab1044f305698551b is 50, key is test_row_0/B:col10/1731960810036/Put/seqid=0 2024-11-18T20:13:30,521 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742172_1348 (size=12001) 2024-11-18T20:13:30,639 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35963 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-18T20:13:30,639 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd. 2024-11-18T20:13:30,639 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd. as already flushing 2024-11-18T20:13:30,639 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd. 2024-11-18T20:13:30,639 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 8eaa77b8f4c3de4047bf5415370077cd, NAME => 'TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:13:30,639 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 8eaa77b8f4c3de4047bf5415370077cd, NAME => 'TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:13:30,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8eaa77b8f4c3de4047bf5415370077cd, NAME => 'TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8eaa77b8f4c3de4047bf5415370077cd, NAME => 'TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:13:30,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-18T20:13:30,726 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:30,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39924 deadline: 1731960870723, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:30,726 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:30,726 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:30,726 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:30,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39882 deadline: 1731960870723, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:30,726 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:13:30,726 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:30,727 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:30,727 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:13:30,738 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:30,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39866 deadline: 1731960870733, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:30,738 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:30,738 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:30,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39834 deadline: 1731960870733, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:30,738 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:30,738 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:13:30,738 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:30,738 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:30,738 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:30,738 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:13:30,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39910 deadline: 1731960870734, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:30,738 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:30,738 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:30,739 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:13:30,792 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35963 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-18T20:13:30,792 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd. 2024-11-18T20:13:30,792 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd. as already flushing 2024-11-18T20:13:30,792 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd. 2024-11-18T20:13:30,792 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 8eaa77b8f4c3de4047bf5415370077cd, NAME => 'TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:13:30,792 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 8eaa77b8f4c3de4047bf5415370077cd, NAME => 'TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:13:30,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8eaa77b8f4c3de4047bf5415370077cd, NAME => 'TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8eaa77b8f4c3de4047bf5415370077cd, NAME => 'TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:13:30,922 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/.tmp/B/cbd2cfe931ca46bab1044f305698551b 2024-11-18T20:13:30,941 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/.tmp/C/39768566c03c4227ad4ea16b60a92a68 is 50, key is test_row_0/C:col10/1731960810036/Put/seqid=0 2024-11-18T20:13:30,944 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742173_1349 (size=12001) 2024-11-18T20:13:30,945 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35963 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-18T20:13:30,945 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd. 2024-11-18T20:13:30,945 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd. as already flushing 2024-11-18T20:13:30,945 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd. 2024-11-18T20:13:30,945 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 8eaa77b8f4c3de4047bf5415370077cd, NAME => 'TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:13:30,945 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 8eaa77b8f4c3de4047bf5415370077cd, NAME => 'TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:13:30,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8eaa77b8f4c3de4047bf5415370077cd, NAME => 'TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8eaa77b8f4c3de4047bf5415370077cd, NAME => 'TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:13:31,096 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35963 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-18T20:13:31,096 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd. 2024-11-18T20:13:31,096 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd. as already flushing 2024-11-18T20:13:31,097 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd. 2024-11-18T20:13:31,097 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 8eaa77b8f4c3de4047bf5415370077cd, NAME => 'TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:13:31,097 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 8eaa77b8f4c3de4047bf5415370077cd, NAME => 'TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:13:31,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8eaa77b8f4c3de4047bf5415370077cd, NAME => 'TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8eaa77b8f4c3de4047bf5415370077cd, NAME => 'TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:13:31,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-18T20:13:31,237 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:31,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39924 deadline: 1731960871233, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:31,237 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:31,238 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:31,238 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:31,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39882 deadline: 1731960871233, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:31,238 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:13:31,238 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:31,238 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:31,238 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:13:31,244 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:31,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39866 deadline: 1731960871243, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:31,244 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:31,245 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:31,245 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:13:31,247 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:31,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39834 deadline: 1731960871243, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:31,247 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:31,247 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:31,247 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:13:31,249 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35963 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-18T20:13:31,249 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd. 2024-11-18T20:13:31,249 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd. as already flushing 2024-11-18T20:13:31,249 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd. 2024-11-18T20:13:31,249 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 8eaa77b8f4c3de4047bf5415370077cd, NAME => 'TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:13:31,249 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 8eaa77b8f4c3de4047bf5415370077cd, NAME => 'TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:13:31,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8eaa77b8f4c3de4047bf5415370077cd, NAME => 'TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8eaa77b8f4c3de4047bf5415370077cd, NAME => 'TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:13:31,256 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:31,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39910 deadline: 1731960871253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:31,256 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:31,256 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:31,257 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:13:31,345 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/.tmp/C/39768566c03c4227ad4ea16b60a92a68 2024-11-18T20:13:31,349 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/.tmp/A/13725551bd2245be8850dcc8a4559260 as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/A/13725551bd2245be8850dcc8a4559260 2024-11-18T20:13:31,352 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/A/13725551bd2245be8850dcc8a4559260, entries=200, sequenceid=16, filesize=38.6 K 2024-11-18T20:13:31,352 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/.tmp/B/cbd2cfe931ca46bab1044f305698551b as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/B/cbd2cfe931ca46bab1044f305698551b 2024-11-18T20:13:31,355 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/B/cbd2cfe931ca46bab1044f305698551b, entries=150, sequenceid=16, filesize=11.7 K 2024-11-18T20:13:31,356 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/.tmp/C/39768566c03c4227ad4ea16b60a92a68 as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/C/39768566c03c4227ad4ea16b60a92a68 2024-11-18T20:13:31,359 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/C/39768566c03c4227ad4ea16b60a92a68, entries=150, sequenceid=16, filesize=11.7 K 2024-11-18T20:13:31,360 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=147.60 KB/151140 for 8eaa77b8f4c3de4047bf5415370077cd in 1321ms, sequenceid=16, compaction requested=false 2024-11-18T20:13:31,360 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8eaa77b8f4c3de4047bf5415370077cd: 2024-11-18T20:13:31,401 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35963 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-18T20:13:31,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd. 2024-11-18T20:13:31,402 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2902): Flushing 8eaa77b8f4c3de4047bf5415370077cd 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-18T20:13:31,409 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411183fbdfe8f0fb848ce8cdb98216cbb6efe_8eaa77b8f4c3de4047bf5415370077cd is 50, key is test_row_0/A:col10/1731960810092/Put/seqid=0 2024-11-18T20:13:31,412 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742174_1350 (size=12154) 2024-11-18T20:13:31,646 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-18T20:13:31,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:31,817 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411183fbdfe8f0fb848ce8cdb98216cbb6efe_8eaa77b8f4c3de4047bf5415370077cd to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411183fbdfe8f0fb848ce8cdb98216cbb6efe_8eaa77b8f4c3de4047bf5415370077cd 2024-11-18T20:13:31,817 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/.tmp/A/882382ffd1fc46d2ab2a16547d84b48b, store: [table=TestAcidGuarantees family=A region=8eaa77b8f4c3de4047bf5415370077cd] 2024-11-18T20:13:31,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/.tmp/A/882382ffd1fc46d2ab2a16547d84b48b is 175, key is test_row_0/A:col10/1731960810092/Put/seqid=0 2024-11-18T20:13:31,822 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742175_1351 (size=30955) 2024-11-18T20:13:32,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-18T20:13:32,223 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=41, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/.tmp/A/882382ffd1fc46d2ab2a16547d84b48b 2024-11-18T20:13:32,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/.tmp/B/54484e6ef51b44bd802d7c46a02134e9 is 50, key is test_row_0/B:col10/1731960810092/Put/seqid=0 2024-11-18T20:13:32,233 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742176_1352 (size=12001) 2024-11-18T20:13:32,234 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=41 (bloomFilter=true), to=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/.tmp/B/54484e6ef51b44bd802d7c46a02134e9 2024-11-18T20:13:32,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/.tmp/C/48f3536a541f42668e7b5a2fbd8164ff is 50, key is test_row_0/C:col10/1731960810092/Put/seqid=0 2024-11-18T20:13:32,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(8855): Flush requested on 8eaa77b8f4c3de4047bf5415370077cd 2024-11-18T20:13:32,249 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd. as already flushing 2024-11-18T20:13:32,250 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742177_1353 (size=12001) 2024-11-18T20:13:32,266 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:32,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39924 deadline: 1731960872260, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:32,267 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:32,267 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:32,267 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:13:32,269 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:32,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39834 deadline: 1731960872263, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:32,269 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:32,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39882 deadline: 1731960872263, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:32,270 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:32,269 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to c0a89b2656d4,35963,1731960718223 for 'test_row_2' in 8eaa77b8f4c3de4047bf5415370077cd of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2172 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-18T20:13:32,270 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:32,270 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:32,270 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:13:32,270 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:32,270 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:13:32,272 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:32,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39866 deadline: 1731960872264, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:32,272 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:32,272 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:32,272 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:13:32,280 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:32,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39910 deadline: 1731960872273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:32,280 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to c0a89b2656d4,35963,1731960718223 for 'test_row_0' in 8eaa77b8f4c3de4047bf5415370077cd of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2187 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-18T20:13:32,280 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:32,280 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:32,280 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:13:32,376 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:32,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39924 deadline: 1731960872373, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:32,377 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:32,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39882 deadline: 1731960872373, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:32,377 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:32,377 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:32,377 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:13:32,377 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:32,377 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:32,377 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:13:32,389 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:32,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39866 deadline: 1731960872383, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:32,390 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:32,390 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:32,390 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:13:32,587 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:32,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39924 deadline: 1731960872583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:32,587 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:32,587 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:32,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39882 deadline: 1731960872583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:32,587 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:32,587 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:13:32,587 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:32,587 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:32,587 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:13:32,595 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:32,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39866 deadline: 1731960872593, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:32,595 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:32,595 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:32,595 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:13:32,650 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=41 (bloomFilter=true), to=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/.tmp/C/48f3536a541f42668e7b5a2fbd8164ff 2024-11-18T20:13:32,654 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/.tmp/A/882382ffd1fc46d2ab2a16547d84b48b as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/A/882382ffd1fc46d2ab2a16547d84b48b 2024-11-18T20:13:32,658 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/A/882382ffd1fc46d2ab2a16547d84b48b, entries=150, sequenceid=41, filesize=30.2 K 2024-11-18T20:13:32,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/.tmp/B/54484e6ef51b44bd802d7c46a02134e9 as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/B/54484e6ef51b44bd802d7c46a02134e9 2024-11-18T20:13:32,661 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/B/54484e6ef51b44bd802d7c46a02134e9, entries=150, sequenceid=41, filesize=11.7 K 2024-11-18T20:13:32,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/.tmp/C/48f3536a541f42668e7b5a2fbd8164ff as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/C/48f3536a541f42668e7b5a2fbd8164ff 2024-11-18T20:13:32,666 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/C/48f3536a541f42668e7b5a2fbd8164ff, entries=150, sequenceid=41, filesize=11.7 K 2024-11-18T20:13:32,667 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 8eaa77b8f4c3de4047bf5415370077cd in 1265ms, sequenceid=41, compaction requested=false 2024-11-18T20:13:32,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2603): Flush status journal for 8eaa77b8f4c3de4047bf5415370077cd: 2024-11-18T20:13:32,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd. 2024-11-18T20:13:32,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=100 2024-11-18T20:13:32,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.HMaster(4169): Remote procedure done, pid=100 2024-11-18T20:13:32,670 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=100, resume processing ppid=99 2024-11-18T20:13:32,670 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=100, ppid=99, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.6430 sec 2024-11-18T20:13:32,671 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=99, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees in 2.6480 sec 2024-11-18T20:13:32,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(8855): Flush requested on 8eaa77b8f4c3de4047bf5415370077cd 2024-11-18T20:13:32,896 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-18T20:13:32,896 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8eaa77b8f4c3de4047bf5415370077cd 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-18T20:13:32,904 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111888c747970e7e4f3c9628cb9a14220339_8eaa77b8f4c3de4047bf5415370077cd is 50, key is test_row_0/A:col10/1731960812262/Put/seqid=0 2024-11-18T20:13:32,907 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742178_1354 (size=12154) 2024-11-18T20:13:32,908 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:32,912 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111888c747970e7e4f3c9628cb9a14220339_8eaa77b8f4c3de4047bf5415370077cd to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111888c747970e7e4f3c9628cb9a14220339_8eaa77b8f4c3de4047bf5415370077cd 2024-11-18T20:13:32,913 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/.tmp/A/bb7a6318ae714a60b837601adf470003, store: [table=TestAcidGuarantees family=A region=8eaa77b8f4c3de4047bf5415370077cd] 2024-11-18T20:13:32,913 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/.tmp/A/bb7a6318ae714a60b837601adf470003 is 175, key is test_row_0/A:col10/1731960812262/Put/seqid=0 2024-11-18T20:13:32,916 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742179_1355 (size=30955) 2024-11-18T20:13:33,007 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:33,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39866 deadline: 1731960872993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:33,007 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:33,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39882 deadline: 1731960872997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:33,007 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:33,007 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:33,008 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:33,008 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:33,008 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:13:33,008 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:13:33,008 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:33,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39924 deadline: 1731960872998, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:33,009 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:33,009 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:33,009 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:13:33,117 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:33,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39866 deadline: 1731960873112, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:33,118 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:33,118 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:33,118 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:33,118 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:13:33,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39882 deadline: 1731960873112, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:33,118 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:33,118 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:33,118 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:33,118 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:13:33,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39924 deadline: 1731960873113, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:33,118 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:33,118 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:33,118 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:13:33,319 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=54, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/.tmp/A/bb7a6318ae714a60b837601adf470003 2024-11-18T20:13:33,327 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/.tmp/B/3c7ab62230bf47c6b8d798c074e521e9 is 50, key is test_row_0/B:col10/1731960812262/Put/seqid=0 2024-11-18T20:13:33,334 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:33,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39866 deadline: 1731960873323, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:33,335 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:33,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39924 deadline: 1731960873323, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:33,335 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:33,335 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:33,335 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:13:33,335 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:33,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39882 deadline: 1731960873323, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:33,335 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:33,335 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:33,335 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:13:33,335 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:33,335 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:33,335 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:13:33,374 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742180_1356 (size=12001) 2024-11-18T20:13:33,375 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=54 (bloomFilter=true), to=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/.tmp/B/3c7ab62230bf47c6b8d798c074e521e9 2024-11-18T20:13:33,398 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/.tmp/C/04e10e915057455dbe4a221d3d1ecbf2 is 50, key is test_row_0/C:col10/1731960812262/Put/seqid=0 2024-11-18T20:13:33,422 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742181_1357 (size=12001) 2024-11-18T20:13:33,647 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:33,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39866 deadline: 1731960873642, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:33,647 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:33,647 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:33,647 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:33,647 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:13:33,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39924 deadline: 1731960873643, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:33,648 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:33,648 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:33,648 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:33,648 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:13:33,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39882 deadline: 1731960873643, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:33,648 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:33,648 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:33,648 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:13:33,823 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=54 (bloomFilter=true), to=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/.tmp/C/04e10e915057455dbe4a221d3d1ecbf2 2024-11-18T20:13:33,829 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/.tmp/A/bb7a6318ae714a60b837601adf470003 as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/A/bb7a6318ae714a60b837601adf470003 2024-11-18T20:13:33,833 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/A/bb7a6318ae714a60b837601adf470003, entries=150, sequenceid=54, filesize=30.2 K 2024-11-18T20:13:33,833 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/.tmp/B/3c7ab62230bf47c6b8d798c074e521e9 as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/B/3c7ab62230bf47c6b8d798c074e521e9 2024-11-18T20:13:33,837 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/B/3c7ab62230bf47c6b8d798c074e521e9, entries=150, sequenceid=54, filesize=11.7 K 2024-11-18T20:13:33,838 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/.tmp/C/04e10e915057455dbe4a221d3d1ecbf2 as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/C/04e10e915057455dbe4a221d3d1ecbf2 2024-11-18T20:13:33,842 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/C/04e10e915057455dbe4a221d3d1ecbf2, entries=150, sequenceid=54, filesize=11.7 K 2024-11-18T20:13:33,842 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for 8eaa77b8f4c3de4047bf5415370077cd in 946ms, sequenceid=54, compaction requested=true 2024-11-18T20:13:33,842 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8eaa77b8f4c3de4047bf5415370077cd: 2024-11-18T20:13:33,843 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8eaa77b8f4c3de4047bf5415370077cd:A, priority=-2147483648, current under compaction store size is 1 2024-11-18T20:13:33,843 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T20:13:33,843 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:13:33,843 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8eaa77b8f4c3de4047bf5415370077cd:B, priority=-2147483648, current under compaction store size is 2 2024-11-18T20:13:33,843 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:13:33,843 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T20:13:33,843 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8eaa77b8f4c3de4047bf5415370077cd:C, priority=-2147483648, current under compaction store size is 3 2024-11-18T20:13:33,843 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T20:13:33,846 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T20:13:33,846 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101459 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T20:13:33,847 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HStore(1541): 8eaa77b8f4c3de4047bf5415370077cd/B is initiating minor compaction (all files) 2024-11-18T20:13:33,847 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HStore(1541): 8eaa77b8f4c3de4047bf5415370077cd/A is initiating minor compaction (all files) 2024-11-18T20:13:33,847 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8eaa77b8f4c3de4047bf5415370077cd/B in TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd. 2024-11-18T20:13:33,847 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8eaa77b8f4c3de4047bf5415370077cd/A in TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd. 2024-11-18T20:13:33,847 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/B/cbd2cfe931ca46bab1044f305698551b, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/B/54484e6ef51b44bd802d7c46a02134e9, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/B/3c7ab62230bf47c6b8d798c074e521e9] into tmpdir=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/.tmp, totalSize=35.2 K 2024-11-18T20:13:33,847 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/A/13725551bd2245be8850dcc8a4559260, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/A/882382ffd1fc46d2ab2a16547d84b48b, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/A/bb7a6318ae714a60b837601adf470003] into tmpdir=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/.tmp, totalSize=99.1 K 2024-11-18T20:13:33,847 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd. 2024-11-18T20:13:33,847 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd. files: [hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/A/13725551bd2245be8850dcc8a4559260, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/A/882382ffd1fc46d2ab2a16547d84b48b, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/A/bb7a6318ae714a60b837601adf470003] 2024-11-18T20:13:33,847 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.Compactor(225): Compacting cbd2cfe931ca46bab1044f305698551b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1731960810027 2024-11-18T20:13:33,847 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] compactions.Compactor(225): Compacting 13725551bd2245be8850dcc8a4559260, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1731960810027 2024-11-18T20:13:33,847 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.Compactor(225): Compacting 54484e6ef51b44bd802d7c46a02134e9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=41, earliestPutTs=1731960810090 2024-11-18T20:13:33,847 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] compactions.Compactor(225): Compacting 882382ffd1fc46d2ab2a16547d84b48b, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=41, earliestPutTs=1731960810090 2024-11-18T20:13:33,847 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.Compactor(225): Compacting 3c7ab62230bf47c6b8d798c074e521e9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1731960812262 2024-11-18T20:13:33,847 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] compactions.Compactor(225): Compacting bb7a6318ae714a60b837601adf470003, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1731960812262 2024-11-18T20:13:33,852 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=8eaa77b8f4c3de4047bf5415370077cd] 2024-11-18T20:13:33,853 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8eaa77b8f4c3de4047bf5415370077cd#B#compaction#308 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:13:33,854 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/.tmp/B/801435fe45ef41bba0024e6545b3ed57 is 50, key is test_row_0/B:col10/1731960812262/Put/seqid=0 2024-11-18T20:13:33,856 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241118ac67887f0c8e4dba8fed3c40a59556cd_8eaa77b8f4c3de4047bf5415370077cd store=[table=TestAcidGuarantees family=A region=8eaa77b8f4c3de4047bf5415370077cd] 2024-11-18T20:13:33,858 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241118ac67887f0c8e4dba8fed3c40a59556cd_8eaa77b8f4c3de4047bf5415370077cd, store=[table=TestAcidGuarantees family=A region=8eaa77b8f4c3de4047bf5415370077cd] 2024-11-18T20:13:33,858 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118ac67887f0c8e4dba8fed3c40a59556cd_8eaa77b8f4c3de4047bf5415370077cd because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=8eaa77b8f4c3de4047bf5415370077cd] 2024-11-18T20:13:33,862 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742182_1358 (size=12104) 2024-11-18T20:13:33,863 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742183_1359 (size=4469) 2024-11-18T20:13:33,864 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8eaa77b8f4c3de4047bf5415370077cd#A#compaction#309 average throughput is 2.04 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:13:33,864 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/.tmp/A/6ea9a7c8ae2d4356a5bad1fa37d0ad60 is 175, key is test_row_0/A:col10/1731960812262/Put/seqid=0 2024-11-18T20:13:33,867 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742184_1360 (size=31058) 2024-11-18T20:13:33,868 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/.tmp/B/801435fe45ef41bba0024e6545b3ed57 as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/B/801435fe45ef41bba0024e6545b3ed57 2024-11-18T20:13:33,874 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8eaa77b8f4c3de4047bf5415370077cd/B of 8eaa77b8f4c3de4047bf5415370077cd into 801435fe45ef41bba0024e6545b3ed57(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:13:33,874 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8eaa77b8f4c3de4047bf5415370077cd: 2024-11-18T20:13:33,874 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., storeName=8eaa77b8f4c3de4047bf5415370077cd/B, priority=13, startTime=1731960813843; duration=0sec 2024-11-18T20:13:33,874 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T20:13:33,874 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8eaa77b8f4c3de4047bf5415370077cd:B 2024-11-18T20:13:33,874 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T20:13:33,875 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T20:13:33,875 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HStore(1541): 8eaa77b8f4c3de4047bf5415370077cd/C is initiating minor compaction (all files) 2024-11-18T20:13:33,875 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8eaa77b8f4c3de4047bf5415370077cd/C in TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd. 2024-11-18T20:13:33,875 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/C/39768566c03c4227ad4ea16b60a92a68, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/C/48f3536a541f42668e7b5a2fbd8164ff, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/C/04e10e915057455dbe4a221d3d1ecbf2] into tmpdir=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/.tmp, totalSize=35.2 K 2024-11-18T20:13:33,875 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.Compactor(225): Compacting 39768566c03c4227ad4ea16b60a92a68, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1731960810027 2024-11-18T20:13:33,876 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.Compactor(225): Compacting 48f3536a541f42668e7b5a2fbd8164ff, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=41, earliestPutTs=1731960810090 2024-11-18T20:13:33,876 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.Compactor(225): Compacting 04e10e915057455dbe4a221d3d1ecbf2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1731960812262 2024-11-18T20:13:33,885 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8eaa77b8f4c3de4047bf5415370077cd#C#compaction#310 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:13:33,886 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/.tmp/C/302b13df853b4d1ca57a92d03443373f is 50, key is test_row_0/C:col10/1731960812262/Put/seqid=0 2024-11-18T20:13:33,893 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742185_1361 (size=12104) 2024-11-18T20:13:34,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(8855): Flush requested on 8eaa77b8f4c3de4047bf5415370077cd 2024-11-18T20:13:34,157 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-18T20:13:34,157 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8eaa77b8f4c3de4047bf5415370077cd 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-18T20:13:34,165 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118092c6c9cedef4d28b7f50dbd4928f09e_8eaa77b8f4c3de4047bf5415370077cd is 50, key is test_row_0/A:col10/1731960814156/Put/seqid=0 2024-11-18T20:13:34,168 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742186_1362 (size=14594) 2024-11-18T20:13:34,189 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:34,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39882 deadline: 1731960874184, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:34,190 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:34,190 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:34,190 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:13:34,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-18T20:13:34,193 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-18T20:13:34,194 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-18T20:13:34,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] procedure2.ProcedureExecutor(1139): Stored pid=101, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees 2024-11-18T20:13:34,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-18T20:13:34,195 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=101, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-18T20:13:34,196 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=101, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-18T20:13:34,196 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=102, ppid=101, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-18T20:13:34,197 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:34,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39924 deadline: 1731960874185, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:34,197 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:34,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39866 deadline: 1731960874186, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:34,197 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:34,197 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:34,197 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:34,197 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:13:34,197 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:34,197 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:13:34,272 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/.tmp/A/6ea9a7c8ae2d4356a5bad1fa37d0ad60 as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/A/6ea9a7c8ae2d4356a5bad1fa37d0ad60 2024-11-18T20:13:34,276 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8eaa77b8f4c3de4047bf5415370077cd/A of 8eaa77b8f4c3de4047bf5415370077cd into 6ea9a7c8ae2d4356a5bad1fa37d0ad60(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:13:34,276 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8eaa77b8f4c3de4047bf5415370077cd: 2024-11-18T20:13:34,276 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., storeName=8eaa77b8f4c3de4047bf5415370077cd/A, priority=13, startTime=1731960813843; duration=0sec 2024-11-18T20:13:34,277 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:13:34,277 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8eaa77b8f4c3de4047bf5415370077cd:A 2024-11-18T20:13:34,291 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:34,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39834 deadline: 1731960874283, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:34,292 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to c0a89b2656d4,35963,1731960718223 for 'test_row_2' in 8eaa77b8f4c3de4047bf5415370077cd of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4194 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-18T20:13:34,292 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:34,292 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:34,292 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:13:34,298 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/.tmp/C/302b13df853b4d1ca57a92d03443373f as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/C/302b13df853b4d1ca57a92d03443373f 2024-11-18T20:13:34,301 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:34,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39882 deadline: 1731960874293, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:34,302 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:34,302 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:34,302 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:13:34,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-18T20:13:34,305 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8eaa77b8f4c3de4047bf5415370077cd/C of 8eaa77b8f4c3de4047bf5415370077cd into 302b13df853b4d1ca57a92d03443373f(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:13:34,305 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8eaa77b8f4c3de4047bf5415370077cd: 2024-11-18T20:13:34,305 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., storeName=8eaa77b8f4c3de4047bf5415370077cd/C, priority=13, startTime=1731960813843; duration=0sec 2024-11-18T20:13:34,305 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:13:34,305 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8eaa77b8f4c3de4047bf5415370077cd:C 2024-11-18T20:13:34,313 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:34,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39924 deadline: 1731960874304, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:34,314 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:34,314 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:34,314 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:34,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39866 deadline: 1731960874304, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:34,314 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:13:34,314 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:34,314 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:34,314 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:34,314 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:13:34,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39910 deadline: 1731960874304, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:34,315 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to c0a89b2656d4,35963,1731960718223 for 'test_row_0' in 8eaa77b8f4c3de4047bf5415370077cd of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4222 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-18T20:13:34,315 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:34,315 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:34,315 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:13:34,348 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35963 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-18T20:13:34,348 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd. 2024-11-18T20:13:34,348 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd. as already flushing 2024-11-18T20:13:34,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd. 2024-11-18T20:13:34,349 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 8eaa77b8f4c3de4047bf5415370077cd, NAME => 'TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:13:34,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 8eaa77b8f4c3de4047bf5415370077cd, NAME => 'TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:13:34,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8eaa77b8f4c3de4047bf5415370077cd, NAME => 'TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8eaa77b8f4c3de4047bf5415370077cd, NAME => 'TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:13:34,502 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35963 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-18T20:13:34,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd. 2024-11-18T20:13:34,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd. as already flushing 2024-11-18T20:13:34,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd. 2024-11-18T20:13:34,502 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 8eaa77b8f4c3de4047bf5415370077cd, NAME => 'TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:13:34,503 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 8eaa77b8f4c3de4047bf5415370077cd, NAME => 'TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:13:34,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8eaa77b8f4c3de4047bf5415370077cd, NAME => 'TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8eaa77b8f4c3de4047bf5415370077cd, NAME => 'TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:13:34,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-18T20:13:34,518 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:34,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39882 deadline: 1731960874513, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:34,519 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:34,519 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:34,519 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:13:34,529 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:34,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39924 deadline: 1731960874523, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:34,529 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:34,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39866 deadline: 1731960874523, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:34,529 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:34,530 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:34,530 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:13:34,530 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:34,530 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:34,530 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:13:34,569 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:34,574 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118092c6c9cedef4d28b7f50dbd4928f09e_8eaa77b8f4c3de4047bf5415370077cd to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118092c6c9cedef4d28b7f50dbd4928f09e_8eaa77b8f4c3de4047bf5415370077cd 2024-11-18T20:13:34,574 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/.tmp/A/98343979eecd4758af30dcd70afe64f5, store: [table=TestAcidGuarantees family=A region=8eaa77b8f4c3de4047bf5415370077cd] 2024-11-18T20:13:34,575 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/.tmp/A/98343979eecd4758af30dcd70afe64f5 is 175, key is test_row_0/A:col10/1731960814156/Put/seqid=0 2024-11-18T20:13:34,578 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742187_1363 (size=39549) 2024-11-18T20:13:34,579 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=79, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/.tmp/A/98343979eecd4758af30dcd70afe64f5 2024-11-18T20:13:34,594 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/.tmp/B/af3a6dea59e543c1991942d24c56605f is 50, key is test_row_0/B:col10/1731960814156/Put/seqid=0 2024-11-18T20:13:34,611 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742188_1364 (size=12001) 2024-11-18T20:13:34,612 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=79 (bloomFilter=true), to=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/.tmp/B/af3a6dea59e543c1991942d24c56605f 2024-11-18T20:13:34,625 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/.tmp/C/8922951d492d486c9f2c499a300ecedc is 50, key is test_row_0/C:col10/1731960814156/Put/seqid=0 2024-11-18T20:13:34,650 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742189_1365 (size=12001) 2024-11-18T20:13:34,651 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=79 (bloomFilter=true), to=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/.tmp/C/8922951d492d486c9f2c499a300ecedc 2024-11-18T20:13:34,655 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35963 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-18T20:13:34,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd. 2024-11-18T20:13:34,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd. as already flushing 2024-11-18T20:13:34,657 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd. 2024-11-18T20:13:34,657 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 8eaa77b8f4c3de4047bf5415370077cd, NAME => 'TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:13:34,657 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 8eaa77b8f4c3de4047bf5415370077cd, NAME => 'TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:13:34,658 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/.tmp/A/98343979eecd4758af30dcd70afe64f5 as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/A/98343979eecd4758af30dcd70afe64f5 2024-11-18T20:13:34,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8eaa77b8f4c3de4047bf5415370077cd, NAME => 'TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8eaa77b8f4c3de4047bf5415370077cd, NAME => 'TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:13:34,666 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/A/98343979eecd4758af30dcd70afe64f5, entries=200, sequenceid=79, filesize=38.6 K 2024-11-18T20:13:34,668 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/.tmp/B/af3a6dea59e543c1991942d24c56605f as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/B/af3a6dea59e543c1991942d24c56605f 2024-11-18T20:13:34,672 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/B/af3a6dea59e543c1991942d24c56605f, entries=150, sequenceid=79, filesize=11.7 K 2024-11-18T20:13:34,672 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/.tmp/C/8922951d492d486c9f2c499a300ecedc as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/C/8922951d492d486c9f2c499a300ecedc 2024-11-18T20:13:34,679 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/C/8922951d492d486c9f2c499a300ecedc, entries=150, sequenceid=79, filesize=11.7 K 2024-11-18T20:13:34,680 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for 8eaa77b8f4c3de4047bf5415370077cd in 523ms, sequenceid=79, compaction requested=false 2024-11-18T20:13:34,680 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8eaa77b8f4c3de4047bf5415370077cd: 2024-11-18T20:13:34,813 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35963 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-18T20:13:34,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd. 2024-11-18T20:13:34,813 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2902): Flushing 8eaa77b8f4c3de4047bf5415370077cd 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-18T20:13:34,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-18T20:13:34,824 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111810150afb837f4a3db3408cd03f824a61_8eaa77b8f4c3de4047bf5415370077cd is 50, key is test_row_0/A:col10/1731960814177/Put/seqid=0 2024-11-18T20:13:34,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(8855): Flush requested on 8eaa77b8f4c3de4047bf5415370077cd 2024-11-18T20:13:34,826 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd. as already flushing 2024-11-18T20:13:34,832 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742190_1366 (size=12154) 2024-11-18T20:13:34,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:34,839 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111810150afb837f4a3db3408cd03f824a61_8eaa77b8f4c3de4047bf5415370077cd to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111810150afb837f4a3db3408cd03f824a61_8eaa77b8f4c3de4047bf5415370077cd 2024-11-18T20:13:34,840 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/.tmp/A/c2eee0845c9c4513bd7be1efb6304d9e, store: [table=TestAcidGuarantees family=A region=8eaa77b8f4c3de4047bf5415370077cd] 2024-11-18T20:13:34,840 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/.tmp/A/c2eee0845c9c4513bd7be1efb6304d9e is 175, key is test_row_0/A:col10/1731960814177/Put/seqid=0 2024-11-18T20:13:34,859 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742191_1367 (size=30955) 2024-11-18T20:13:34,859 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=93, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/.tmp/A/c2eee0845c9c4513bd7be1efb6304d9e 2024-11-18T20:13:34,867 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/.tmp/B/a448cbc67a664c2e9fec5ec7bf835c60 is 50, key is test_row_0/B:col10/1731960814177/Put/seqid=0 2024-11-18T20:13:34,884 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742192_1368 (size=12001) 2024-11-18T20:13:34,885 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=93 (bloomFilter=true), to=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/.tmp/B/a448cbc67a664c2e9fec5ec7bf835c60 2024-11-18T20:13:34,894 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/.tmp/C/3c284c0c7bb54f7086a229dab2f25984 is 50, key is test_row_0/C:col10/1731960814177/Put/seqid=0 2024-11-18T20:13:34,899 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742193_1369 (size=12001) 2024-11-18T20:13:34,901 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=93 (bloomFilter=true), to=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/.tmp/C/3c284c0c7bb54f7086a229dab2f25984 2024-11-18T20:13:34,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/.tmp/A/c2eee0845c9c4513bd7be1efb6304d9e as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/A/c2eee0845c9c4513bd7be1efb6304d9e 2024-11-18T20:13:34,910 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/A/c2eee0845c9c4513bd7be1efb6304d9e, entries=150, sequenceid=93, filesize=30.2 K 2024-11-18T20:13:34,910 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/.tmp/B/a448cbc67a664c2e9fec5ec7bf835c60 as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/B/a448cbc67a664c2e9fec5ec7bf835c60 2024-11-18T20:13:34,918 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/B/a448cbc67a664c2e9fec5ec7bf835c60, entries=150, sequenceid=93, filesize=11.7 K 2024-11-18T20:13:34,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/.tmp/C/3c284c0c7bb54f7086a229dab2f25984 as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/C/3c284c0c7bb54f7086a229dab2f25984 2024-11-18T20:13:34,922 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:34,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39924 deadline: 1731960874910, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:34,922 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:34,922 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:34,922 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:34,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39866 deadline: 1731960874913, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:34,922 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:13:34,922 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/C/3c284c0c7bb54f7086a229dab2f25984, entries=150, sequenceid=93, filesize=11.7 K 2024-11-18T20:13:34,922 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:34,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39882 deadline: 1731960874921, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:34,923 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:34,923 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:34,923 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:13:34,924 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 8eaa77b8f4c3de4047bf5415370077cd in 110ms, sequenceid=93, compaction requested=true 2024-11-18T20:13:34,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2603): Flush status journal for 8eaa77b8f4c3de4047bf5415370077cd: 2024-11-18T20:13:34,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd. 2024-11-18T20:13:34,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=102 2024-11-18T20:13:34,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.HMaster(4169): Remote procedure done, pid=102 2024-11-18T20:13:34,924 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:34,924 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:34,924 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:13:34,926 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=102, resume processing ppid=101 2024-11-18T20:13:34,926 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=102, ppid=101, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 729 msec 2024-11-18T20:13:34,928 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=101, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees in 733 msec 2024-11-18T20:13:35,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(8855): Flush requested on 8eaa77b8f4c3de4047bf5415370077cd 2024-11-18T20:13:35,037 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-18T20:13:35,037 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8eaa77b8f4c3de4047bf5415370077cd 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-18T20:13:35,044 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118a28a3f9035c24184a43a33d30e7ecefb_8eaa77b8f4c3de4047bf5415370077cd is 50, key is test_row_0/A:col10/1731960815036/Put/seqid=0 2024-11-18T20:13:35,047 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742194_1370 (size=14594) 2024-11-18T20:13:35,065 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:35,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39924 deadline: 1731960875054, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:35,066 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:35,066 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:35,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39882 deadline: 1731960875055, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:35,066 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:35,066 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:13:35,066 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:35,066 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:35,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39866 deadline: 1731960875060, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:35,066 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:35,066 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:13:35,066 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:35,066 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:35,066 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:13:35,180 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:35,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39924 deadline: 1731960875173, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:35,181 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:35,181 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:35,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39882 deadline: 1731960875173, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:35,181 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:35,181 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:13:35,181 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:35,181 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:35,181 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:35,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39866 deadline: 1731960875173, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:35,181 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:13:35,182 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:35,182 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:35,182 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:13:35,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-18T20:13:35,333 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-18T20:13:35,333 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-18T20:13:35,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] procedure2.ProcedureExecutor(1139): Stored pid=103, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees 2024-11-18T20:13:35,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-18T20:13:35,335 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=103, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-18T20:13:35,335 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=103, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-18T20:13:35,336 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=104, ppid=103, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-18T20:13:35,403 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:35,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39924 deadline: 1731960875393, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:35,403 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:35,403 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:35,403 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:35,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39882 deadline: 1731960875393, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:35,403 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:13:35,404 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:35,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39866 deadline: 1731960875393, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:35,404 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:35,404 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:35,404 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:13:35,405 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:35,405 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:35,405 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:13:35,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-18T20:13:35,458 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:35,464 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118a28a3f9035c24184a43a33d30e7ecefb_8eaa77b8f4c3de4047bf5415370077cd to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118a28a3f9035c24184a43a33d30e7ecefb_8eaa77b8f4c3de4047bf5415370077cd 2024-11-18T20:13:35,465 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/.tmp/A/98e854168bb1409588a759becb8a64d9, store: [table=TestAcidGuarantees family=A region=8eaa77b8f4c3de4047bf5415370077cd] 2024-11-18T20:13:35,466 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/.tmp/A/98e854168bb1409588a759becb8a64d9 is 175, key is test_row_0/A:col10/1731960815036/Put/seqid=0 2024-11-18T20:13:35,470 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742195_1371 (size=39549) 2024-11-18T20:13:35,487 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35963 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-18T20:13:35,488 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd. 2024-11-18T20:13:35,488 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd. as already flushing 2024-11-18T20:13:35,488 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd. 2024-11-18T20:13:35,488 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 8eaa77b8f4c3de4047bf5415370077cd, NAME => 'TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:13:35,488 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 8eaa77b8f4c3de4047bf5415370077cd, NAME => 'TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:13:35,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8eaa77b8f4c3de4047bf5415370077cd, NAME => 'TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8eaa77b8f4c3de4047bf5415370077cd, NAME => 'TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:13:35,641 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35963 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-18T20:13:35,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd. 2024-11-18T20:13:35,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd. as already flushing 2024-11-18T20:13:35,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd. 2024-11-18T20:13:35,642 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 8eaa77b8f4c3de4047bf5415370077cd, NAME => 'TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:13:35,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 8eaa77b8f4c3de4047bf5415370077cd, NAME => 'TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:13:35,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8eaa77b8f4c3de4047bf5415370077cd, NAME => 'TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8eaa77b8f4c3de4047bf5415370077cd, NAME => 'TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:13:35,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-18T20:13:35,720 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:35,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39924 deadline: 1731960875713, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:35,720 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:35,720 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:35,721 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:35,721 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:13:35,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39882 deadline: 1731960875713, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:35,721 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:35,721 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:35,721 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:13:35,721 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:35,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39866 deadline: 1731960875713, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:35,721 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:35,721 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:35,721 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:13:35,799 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35963 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-18T20:13:35,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd. 2024-11-18T20:13:35,800 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd. as already flushing 2024-11-18T20:13:35,800 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd. 2024-11-18T20:13:35,800 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 8eaa77b8f4c3de4047bf5415370077cd, NAME => 'TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:13:35,800 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 8eaa77b8f4c3de4047bf5415370077cd, NAME => 'TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:13:35,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8eaa77b8f4c3de4047bf5415370077cd, NAME => 'TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8eaa77b8f4c3de4047bf5415370077cd, NAME => 'TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:13:35,871 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=118, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/.tmp/A/98e854168bb1409588a759becb8a64d9 2024-11-18T20:13:35,877 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/.tmp/B/0a6c66df5a7b447daec65d9693b6176b is 50, key is test_row_0/B:col10/1731960815036/Put/seqid=0 2024-11-18T20:13:35,888 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742196_1372 (size=12001) 2024-11-18T20:13:35,952 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35963 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-18T20:13:35,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd. 2024-11-18T20:13:35,953 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd. as already flushing 2024-11-18T20:13:35,953 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd. 2024-11-18T20:13:35,953 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 8eaa77b8f4c3de4047bf5415370077cd, NAME => 'TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:13:35,953 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 8eaa77b8f4c3de4047bf5415370077cd, NAME => 'TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:13:35,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8eaa77b8f4c3de4047bf5415370077cd, NAME => 'TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8eaa77b8f4c3de4047bf5415370077cd, NAME => 'TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:13:35,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-18T20:13:36,106 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35963 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-18T20:13:36,106 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd. 2024-11-18T20:13:36,106 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd. as already flushing 2024-11-18T20:13:36,106 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd. 2024-11-18T20:13:36,106 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 8eaa77b8f4c3de4047bf5415370077cd, NAME => 'TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:13:36,106 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 8eaa77b8f4c3de4047bf5415370077cd, NAME => 'TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:13:36,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8eaa77b8f4c3de4047bf5415370077cd, NAME => 'TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8eaa77b8f4c3de4047bf5415370077cd, NAME => 'TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:13:36,239 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:36,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39924 deadline: 1731960876233, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:36,240 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:36,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39882 deadline: 1731960876234, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:36,240 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:36,240 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:36,240 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:13:36,240 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:36,240 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:36,240 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:36,240 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:13:36,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39866 deadline: 1731960876234, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:36,240 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:36,240 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:36,240 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:13:36,259 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35963 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-18T20:13:36,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd. 2024-11-18T20:13:36,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd. as already flushing 2024-11-18T20:13:36,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd. 2024-11-18T20:13:36,259 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 8eaa77b8f4c3de4047bf5415370077cd, NAME => 'TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:13:36,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 8eaa77b8f4c3de4047bf5415370077cd, NAME => 'TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:13:36,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8eaa77b8f4c3de4047bf5415370077cd, NAME => 'TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8eaa77b8f4c3de4047bf5415370077cd, NAME => 'TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:13:36,289 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=118 (bloomFilter=true), to=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/.tmp/B/0a6c66df5a7b447daec65d9693b6176b 2024-11-18T20:13:36,295 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/.tmp/C/5910ecbcd40949fd8e61cad60c7f0aeb is 50, key is test_row_0/C:col10/1731960815036/Put/seqid=0 2024-11-18T20:13:36,299 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742197_1373 (size=12001) 2024-11-18T20:13:36,411 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35963 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-18T20:13:36,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd. 2024-11-18T20:13:36,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd. as already flushing 2024-11-18T20:13:36,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd. 2024-11-18T20:13:36,412 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 8eaa77b8f4c3de4047bf5415370077cd, NAME => 'TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:13:36,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 8eaa77b8f4c3de4047bf5415370077cd, NAME => 'TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:13:36,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8eaa77b8f4c3de4047bf5415370077cd, NAME => 'TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8eaa77b8f4c3de4047bf5415370077cd, NAME => 'TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:13:36,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-18T20:13:36,566 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35963 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-18T20:13:36,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd. 2024-11-18T20:13:36,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd. as already flushing 2024-11-18T20:13:36,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd. 2024-11-18T20:13:36,566 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 8eaa77b8f4c3de4047bf5415370077cd, NAME => 'TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:13:36,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 8eaa77b8f4c3de4047bf5415370077cd, NAME => 'TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:13:36,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8eaa77b8f4c3de4047bf5415370077cd, NAME => 'TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8eaa77b8f4c3de4047bf5415370077cd, NAME => 'TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:13:36,699 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=118 (bloomFilter=true), to=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/.tmp/C/5910ecbcd40949fd8e61cad60c7f0aeb 2024-11-18T20:13:36,703 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/.tmp/A/98e854168bb1409588a759becb8a64d9 as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/A/98e854168bb1409588a759becb8a64d9 2024-11-18T20:13:36,706 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/A/98e854168bb1409588a759becb8a64d9, entries=200, sequenceid=118, filesize=38.6 K 2024-11-18T20:13:36,707 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/.tmp/B/0a6c66df5a7b447daec65d9693b6176b as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/B/0a6c66df5a7b447daec65d9693b6176b 2024-11-18T20:13:36,710 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/B/0a6c66df5a7b447daec65d9693b6176b, entries=150, sequenceid=118, filesize=11.7 K 2024-11-18T20:13:36,710 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/.tmp/C/5910ecbcd40949fd8e61cad60c7f0aeb as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/C/5910ecbcd40949fd8e61cad60c7f0aeb 2024-11-18T20:13:36,713 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/C/5910ecbcd40949fd8e61cad60c7f0aeb, entries=150, sequenceid=118, filesize=11.7 K 2024-11-18T20:13:36,714 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 8eaa77b8f4c3de4047bf5415370077cd in 1676ms, sequenceid=118, compaction requested=true 2024-11-18T20:13:36,714 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8eaa77b8f4c3de4047bf5415370077cd: 2024-11-18T20:13:36,714 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8eaa77b8f4c3de4047bf5415370077cd:A, priority=-2147483648, current under compaction store size is 1 2024-11-18T20:13:36,714 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-18T20:13:36,714 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:13:36,714 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-18T20:13:36,714 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8eaa77b8f4c3de4047bf5415370077cd:B, priority=-2147483648, current under compaction store size is 2 2024-11-18T20:13:36,714 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:13:36,714 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8eaa77b8f4c3de4047bf5415370077cd:C, priority=-2147483648, current under compaction store size is 3 2024-11-18T20:13:36,714 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T20:13:36,715 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-18T20:13:36,715 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 141111 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-18T20:13:36,715 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HStore(1541): 8eaa77b8f4c3de4047bf5415370077cd/A is initiating minor compaction (all files) 2024-11-18T20:13:36,715 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HStore(1541): 8eaa77b8f4c3de4047bf5415370077cd/B is initiating minor compaction (all files) 2024-11-18T20:13:36,715 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8eaa77b8f4c3de4047bf5415370077cd/B in TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd. 2024-11-18T20:13:36,715 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8eaa77b8f4c3de4047bf5415370077cd/A in TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd. 2024-11-18T20:13:36,715 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/B/801435fe45ef41bba0024e6545b3ed57, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/B/af3a6dea59e543c1991942d24c56605f, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/B/a448cbc67a664c2e9fec5ec7bf835c60, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/B/0a6c66df5a7b447daec65d9693b6176b] into tmpdir=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/.tmp, totalSize=47.0 K 2024-11-18T20:13:36,715 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/A/6ea9a7c8ae2d4356a5bad1fa37d0ad60, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/A/98343979eecd4758af30dcd70afe64f5, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/A/c2eee0845c9c4513bd7be1efb6304d9e, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/A/98e854168bb1409588a759becb8a64d9] into tmpdir=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/.tmp, totalSize=137.8 K 2024-11-18T20:13:36,715 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd. 2024-11-18T20:13:36,715 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd. files: [hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/A/6ea9a7c8ae2d4356a5bad1fa37d0ad60, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/A/98343979eecd4758af30dcd70afe64f5, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/A/c2eee0845c9c4513bd7be1efb6304d9e, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/A/98e854168bb1409588a759becb8a64d9] 2024-11-18T20:13:36,716 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.Compactor(225): Compacting 801435fe45ef41bba0024e6545b3ed57, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1731960812262 2024-11-18T20:13:36,716 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6ea9a7c8ae2d4356a5bad1fa37d0ad60, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1731960812262 2024-11-18T20:13:36,716 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] compactions.Compactor(225): Compacting 98343979eecd4758af30dcd70afe64f5, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=79, earliestPutTs=1731960812981 2024-11-18T20:13:36,716 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.Compactor(225): Compacting af3a6dea59e543c1991942d24c56605f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=79, earliestPutTs=1731960812981 2024-11-18T20:13:36,716 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.Compactor(225): Compacting a448cbc67a664c2e9fec5ec7bf835c60, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1731960814177 2024-11-18T20:13:36,717 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] compactions.Compactor(225): Compacting c2eee0845c9c4513bd7be1efb6304d9e, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1731960814177 2024-11-18T20:13:36,717 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.Compactor(225): Compacting 0a6c66df5a7b447daec65d9693b6176b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1731960814908 2024-11-18T20:13:36,717 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] compactions.Compactor(225): Compacting 98e854168bb1409588a759becb8a64d9, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1731960814908 2024-11-18T20:13:36,718 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35963 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-18T20:13:36,719 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd. 2024-11-18T20:13:36,719 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2902): Flushing 8eaa77b8f4c3de4047bf5415370077cd 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-18T20:13:36,728 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=8eaa77b8f4c3de4047bf5415370077cd] 2024-11-18T20:13:36,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118bf411a600a6f4d5b945aae711923a324_8eaa77b8f4c3de4047bf5415370077cd is 50, key is test_row_0/A:col10/1731960815059/Put/seqid=0 2024-11-18T20:13:36,736 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8eaa77b8f4c3de4047bf5415370077cd#B#compaction#322 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:13:36,736 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/.tmp/B/2e3e7c6ea78646e49f09afff72190a85 is 50, key is test_row_0/B:col10/1731960815036/Put/seqid=0 2024-11-18T20:13:36,739 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241118b926dbc61a694d77a577278602f5a35b_8eaa77b8f4c3de4047bf5415370077cd store=[table=TestAcidGuarantees family=A region=8eaa77b8f4c3de4047bf5415370077cd] 2024-11-18T20:13:36,742 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241118b926dbc61a694d77a577278602f5a35b_8eaa77b8f4c3de4047bf5415370077cd, store=[table=TestAcidGuarantees family=A region=8eaa77b8f4c3de4047bf5415370077cd] 2024-11-18T20:13:36,742 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118b926dbc61a694d77a577278602f5a35b_8eaa77b8f4c3de4047bf5415370077cd because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=8eaa77b8f4c3de4047bf5415370077cd] 2024-11-18T20:13:36,749 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742198_1374 (size=12241) 2024-11-18T20:13:36,757 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742199_1375 (size=12154) 2024-11-18T20:13:36,758 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:36,761 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118bf411a600a6f4d5b945aae711923a324_8eaa77b8f4c3de4047bf5415370077cd to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118bf411a600a6f4d5b945aae711923a324_8eaa77b8f4c3de4047bf5415370077cd 2024-11-18T20:13:36,761 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/.tmp/A/586b29f729354327b15ca11a6051cc77, store: [table=TestAcidGuarantees family=A region=8eaa77b8f4c3de4047bf5415370077cd] 2024-11-18T20:13:36,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/.tmp/A/586b29f729354327b15ca11a6051cc77 is 175, key is test_row_0/A:col10/1731960815059/Put/seqid=0 2024-11-18T20:13:36,766 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742200_1376 (size=4469) 2024-11-18T20:13:36,767 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8eaa77b8f4c3de4047bf5415370077cd#A#compaction#320 average throughput is 0.63 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:13:36,768 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/.tmp/A/95ce812f00ed4f3cbbdfb0528d61f7d1 is 175, key is test_row_0/A:col10/1731960815036/Put/seqid=0 2024-11-18T20:13:36,774 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742201_1377 (size=30955) 2024-11-18T20:13:36,775 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=129, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/.tmp/A/586b29f729354327b15ca11a6051cc77 2024-11-18T20:13:36,778 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742202_1378 (size=31195) 2024-11-18T20:13:36,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/.tmp/B/8ea784f2d434473fb944939472970ad3 is 50, key is test_row_0/B:col10/1731960815059/Put/seqid=0 2024-11-18T20:13:36,786 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/.tmp/A/95ce812f00ed4f3cbbdfb0528d61f7d1 as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/A/95ce812f00ed4f3cbbdfb0528d61f7d1 2024-11-18T20:13:36,788 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742203_1379 (size=12001) 2024-11-18T20:13:36,791 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 8eaa77b8f4c3de4047bf5415370077cd/A of 8eaa77b8f4c3de4047bf5415370077cd into 95ce812f00ed4f3cbbdfb0528d61f7d1(size=30.5 K), total size for store is 30.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:13:36,791 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8eaa77b8f4c3de4047bf5415370077cd: 2024-11-18T20:13:36,791 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., storeName=8eaa77b8f4c3de4047bf5415370077cd/A, priority=12, startTime=1731960816714; duration=0sec 2024-11-18T20:13:36,791 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T20:13:36,791 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8eaa77b8f4c3de4047bf5415370077cd:A 2024-11-18T20:13:36,791 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-18T20:13:36,792 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-18T20:13:36,792 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HStore(1541): 8eaa77b8f4c3de4047bf5415370077cd/C is initiating minor compaction (all files) 2024-11-18T20:13:36,792 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8eaa77b8f4c3de4047bf5415370077cd/C in TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd. 2024-11-18T20:13:36,793 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/C/302b13df853b4d1ca57a92d03443373f, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/C/8922951d492d486c9f2c499a300ecedc, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/C/3c284c0c7bb54f7086a229dab2f25984, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/C/5910ecbcd40949fd8e61cad60c7f0aeb] into tmpdir=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/.tmp, totalSize=47.0 K 2024-11-18T20:13:36,793 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] compactions.Compactor(225): Compacting 302b13df853b4d1ca57a92d03443373f, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1731960812262 2024-11-18T20:13:36,793 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8922951d492d486c9f2c499a300ecedc, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=79, earliestPutTs=1731960812981 2024-11-18T20:13:36,793 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3c284c0c7bb54f7086a229dab2f25984, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1731960814177 2024-11-18T20:13:36,794 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5910ecbcd40949fd8e61cad60c7f0aeb, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1731960814908 2024-11-18T20:13:36,801 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8eaa77b8f4c3de4047bf5415370077cd#C#compaction#324 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:13:36,801 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/.tmp/C/488e247322a34eb9a7675567f725bc26 is 50, key is test_row_0/C:col10/1731960815036/Put/seqid=0 2024-11-18T20:13:36,804 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742204_1380 (size=12241) 2024-11-18T20:13:37,153 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/.tmp/B/2e3e7c6ea78646e49f09afff72190a85 as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/B/2e3e7c6ea78646e49f09afff72190a85 2024-11-18T20:13:37,157 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 8eaa77b8f4c3de4047bf5415370077cd/B of 8eaa77b8f4c3de4047bf5415370077cd into 2e3e7c6ea78646e49f09afff72190a85(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:13:37,157 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8eaa77b8f4c3de4047bf5415370077cd: 2024-11-18T20:13:37,157 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., storeName=8eaa77b8f4c3de4047bf5415370077cd/B, priority=12, startTime=1731960816714; duration=0sec 2024-11-18T20:13:37,157 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:13:37,157 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8eaa77b8f4c3de4047bf5415370077cd:B 2024-11-18T20:13:37,188 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=129 (bloomFilter=true), to=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/.tmp/B/8ea784f2d434473fb944939472970ad3 2024-11-18T20:13:37,194 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/.tmp/C/130ab60e382345f4afd0d49dfa7d11c0 is 50, key is test_row_0/C:col10/1731960815059/Put/seqid=0 2024-11-18T20:13:37,209 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742205_1381 (size=12001) 2024-11-18T20:13:37,209 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=129 (bloomFilter=true), to=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/.tmp/C/130ab60e382345f4afd0d49dfa7d11c0 2024-11-18T20:13:37,213 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/.tmp/C/488e247322a34eb9a7675567f725bc26 as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/C/488e247322a34eb9a7675567f725bc26 2024-11-18T20:13:37,215 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/.tmp/A/586b29f729354327b15ca11a6051cc77 as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/A/586b29f729354327b15ca11a6051cc77 2024-11-18T20:13:37,218 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 8eaa77b8f4c3de4047bf5415370077cd/C of 8eaa77b8f4c3de4047bf5415370077cd into 488e247322a34eb9a7675567f725bc26(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:13:37,218 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8eaa77b8f4c3de4047bf5415370077cd: 2024-11-18T20:13:37,218 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., storeName=8eaa77b8f4c3de4047bf5415370077cd/C, priority=12, startTime=1731960816714; duration=0sec 2024-11-18T20:13:37,218 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:13:37,218 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8eaa77b8f4c3de4047bf5415370077cd:C 2024-11-18T20:13:37,219 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/A/586b29f729354327b15ca11a6051cc77, entries=150, sequenceid=129, filesize=30.2 K 2024-11-18T20:13:37,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/.tmp/B/8ea784f2d434473fb944939472970ad3 as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/B/8ea784f2d434473fb944939472970ad3 2024-11-18T20:13:37,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,222 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/B/8ea784f2d434473fb944939472970ad3, entries=150, sequenceid=129, filesize=11.7 K 2024-11-18T20:13:37,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/.tmp/C/130ab60e382345f4afd0d49dfa7d11c0 as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/C/130ab60e382345f4afd0d49dfa7d11c0 2024-11-18T20:13:37,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,227 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/C/130ab60e382345f4afd0d49dfa7d11c0, entries=150, sequenceid=129, filesize=11.7 K 2024-11-18T20:13:37,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,227 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=0 B/0 for 8eaa77b8f4c3de4047bf5415370077cd in 508ms, sequenceid=129, compaction requested=false 2024-11-18T20:13:37,228 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2603): Flush status journal for 8eaa77b8f4c3de4047bf5415370077cd: 2024-11-18T20:13:37,228 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd. 2024-11-18T20:13:37,228 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=104 2024-11-18T20:13:37,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.HMaster(4169): Remote procedure done, pid=104 2024-11-18T20:13:37,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,230 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=104, resume processing ppid=103 2024-11-18T20:13:37,230 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=104, ppid=103, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8930 sec 2024-11-18T20:13:37,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,232 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=103, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees in 1.8980 sec 2024-11-18T20:13:37,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(8855): Flush requested on 8eaa77b8f4c3de4047bf5415370077cd 2024-11-18T20:13:37,376 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-18T20:13:37,376 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8eaa77b8f4c3de4047bf5415370077cd 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-18T20:13:37,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,383 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411181a87ceb9cd434ddf8ac4006d2e920eb9_8eaa77b8f4c3de4047bf5415370077cd is 50, key is test_row_0/A:col10/1731960817363/Put/seqid=0 2024-11-18T20:13:37,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,398 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742206_1382 (size=19774) 2024-11-18T20:13:37,448 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:37,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39882 deadline: 1731960877445, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:37,448 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:37,448 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:37,448 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:13:37,454 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:37,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39924 deadline: 1731960877446, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:37,454 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:37,454 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:37,454 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:13:37,455 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:37,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39866 deadline: 1731960877447, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:37,455 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:37,455 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:37,455 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:13:37,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-18T20:13:37,483 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-18T20:13:37,483 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-18T20:13:37,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] procedure2.ProcedureExecutor(1139): Stored pid=105, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees 2024-11-18T20:13:37,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-18T20:13:37,485 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=105, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-18T20:13:37,485 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=105, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-18T20:13:37,486 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=106, ppid=105, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-18T20:13:37,558 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:37,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39882 deadline: 1731960877553, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:37,558 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:37,558 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:37,558 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:13:37,566 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:37,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39924 deadline: 1731960877562, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:37,567 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:37,567 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:37,567 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:37,567 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:13:37,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39866 deadline: 1731960877562, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:37,567 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:37,567 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:37,567 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:13:37,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-18T20:13:37,636 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35963 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-18T20:13:37,636 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd. 2024-11-18T20:13:37,636 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd. as already flushing 2024-11-18T20:13:37,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd. 2024-11-18T20:13:37,637 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 8eaa77b8f4c3de4047bf5415370077cd, NAME => 'TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:13:37,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 8eaa77b8f4c3de4047bf5415370077cd, NAME => 'TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:13:37,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8eaa77b8f4c3de4047bf5415370077cd, NAME => 'TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8eaa77b8f4c3de4047bf5415370077cd, NAME => 'TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:13:37,770 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:37,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39882 deadline: 1731960877763, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:37,770 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:37,770 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:37,770 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:13:37,783 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:37,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39924 deadline: 1731960877773, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:37,784 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:37,784 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:37,784 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:37,784 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:13:37,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39866 deadline: 1731960877773, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:37,784 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:37,784 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:37,784 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:13:37,789 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35963 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-18T20:13:37,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd. 2024-11-18T20:13:37,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd. as already flushing 2024-11-18T20:13:37,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd. 2024-11-18T20:13:37,789 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 8eaa77b8f4c3de4047bf5415370077cd, NAME => 'TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:13:37,790 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 8eaa77b8f4c3de4047bf5415370077cd, NAME => 'TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:13:37,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8eaa77b8f4c3de4047bf5415370077cd, NAME => 'TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8eaa77b8f4c3de4047bf5415370077cd, NAME => 'TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:13:37,798 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:37,801 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411181a87ceb9cd434ddf8ac4006d2e920eb9_8eaa77b8f4c3de4047bf5415370077cd to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411181a87ceb9cd434ddf8ac4006d2e920eb9_8eaa77b8f4c3de4047bf5415370077cd 2024-11-18T20:13:37,802 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/.tmp/A/9359688205b24faf89fd3d55818f214f, store: [table=TestAcidGuarantees family=A region=8eaa77b8f4c3de4047bf5415370077cd] 2024-11-18T20:13:37,802 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/.tmp/A/9359688205b24faf89fd3d55818f214f is 175, key is test_row_0/A:col10/1731960817363/Put/seqid=0 2024-11-18T20:13:37,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-18T20:13:37,808 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742207_1383 (size=57033) 2024-11-18T20:13:37,922 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-18T20:13:37,922 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-11-18T20:13:37,942 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35963 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-18T20:13:37,942 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd. 2024-11-18T20:13:37,942 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd. as already flushing 2024-11-18T20:13:37,942 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd. 2024-11-18T20:13:37,942 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 8eaa77b8f4c3de4047bf5415370077cd, NAME => 'TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:13:37,942 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 8eaa77b8f4c3de4047bf5415370077cd, NAME => 'TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:13:37,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8eaa77b8f4c3de4047bf5415370077cd, NAME => 'TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8eaa77b8f4c3de4047bf5415370077cd, NAME => 'TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:13:38,088 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:38,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39882 deadline: 1731960878083, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:38,089 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:38,089 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:38,089 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:13:38,094 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35963 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-18T20:13:38,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd. 2024-11-18T20:13:38,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd. as already flushing 2024-11-18T20:13:38,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd. 2024-11-18T20:13:38,095 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 8eaa77b8f4c3de4047bf5415370077cd, NAME => 'TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:13:38,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 8eaa77b8f4c3de4047bf5415370077cd, NAME => 'TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:13:38,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8eaa77b8f4c3de4047bf5415370077cd, NAME => 'TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8eaa77b8f4c3de4047bf5415370077cd, NAME => 'TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:13:38,097 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:38,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39924 deadline: 1731960878093, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:38,097 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:38,097 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:38,097 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:38,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39866 deadline: 1731960878093, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:38,097 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:13:38,098 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:38,098 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:38,098 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:13:38,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-18T20:13:38,209 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=143, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/.tmp/A/9359688205b24faf89fd3d55818f214f 2024-11-18T20:13:38,215 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/.tmp/B/279c46f2a62d4ff4862d72bf147d5229 is 50, key is test_row_0/B:col10/1731960817363/Put/seqid=0 2024-11-18T20:13:38,222 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742208_1384 (size=12151) 2024-11-18T20:13:38,247 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35963 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-18T20:13:38,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd. 2024-11-18T20:13:38,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd. as already flushing 2024-11-18T20:13:38,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd. 2024-11-18T20:13:38,247 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 8eaa77b8f4c3de4047bf5415370077cd, NAME => 'TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:13:38,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 8eaa77b8f4c3de4047bf5415370077cd, NAME => 'TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:13:38,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8eaa77b8f4c3de4047bf5415370077cd, NAME => 'TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8eaa77b8f4c3de4047bf5415370077cd, NAME => 'TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:13:38,329 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:38,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39910 deadline: 1731960878323, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:38,329 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to c0a89b2656d4,35963,1731960718223 for 'test_row_0' in 8eaa77b8f4c3de4047bf5415370077cd of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8236 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-18T20:13:38,330 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:38,330 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:38,330 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:13:38,339 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:38,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39834 deadline: 1731960878333, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:38,339 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to c0a89b2656d4,35963,1731960718223 for 'test_row_2' in 8eaa77b8f4c3de4047bf5415370077cd of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8242 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-18T20:13:38,339 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:38,339 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:38,340 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:13:38,399 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35963 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-18T20:13:38,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd. 2024-11-18T20:13:38,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd. as already flushing 2024-11-18T20:13:38,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd. 2024-11-18T20:13:38,400 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 8eaa77b8f4c3de4047bf5415370077cd, NAME => 'TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:13:38,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 8eaa77b8f4c3de4047bf5415370077cd, NAME => 'TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:13:38,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8eaa77b8f4c3de4047bf5415370077cd, NAME => 'TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8eaa77b8f4c3de4047bf5415370077cd, NAME => 'TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:13:38,552 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35963 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-18T20:13:38,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd. 2024-11-18T20:13:38,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd. as already flushing 2024-11-18T20:13:38,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd. 2024-11-18T20:13:38,553 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 8eaa77b8f4c3de4047bf5415370077cd, NAME => 'TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:13:38,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 8eaa77b8f4c3de4047bf5415370077cd, NAME => 'TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:13:38,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8eaa77b8f4c3de4047bf5415370077cd, NAME => 'TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8eaa77b8f4c3de4047bf5415370077cd, NAME => 'TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:13:38,599 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:38,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39882 deadline: 1731960878593, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:38,600 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:38,600 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:38,600 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:13:38,608 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:38,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39924 deadline: 1731960878603, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:38,608 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:38,608 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:38,608 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:13:38,619 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:38,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39866 deadline: 1731960878613, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:38,620 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:38,620 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:38,620 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:13:38,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-18T20:13:38,623 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=143 (bloomFilter=true), to=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/.tmp/B/279c46f2a62d4ff4862d72bf147d5229 2024-11-18T20:13:38,629 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/.tmp/C/67b001ee8443430dbb148881a5bf85df is 50, key is test_row_0/C:col10/1731960817363/Put/seqid=0 2024-11-18T20:13:38,634 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742209_1385 (size=12151) 2024-11-18T20:13:38,705 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35963 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-18T20:13:38,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd. 2024-11-18T20:13:38,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd. as already flushing 2024-11-18T20:13:38,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd. 2024-11-18T20:13:38,705 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 8eaa77b8f4c3de4047bf5415370077cd, NAME => 'TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:13:38,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 8eaa77b8f4c3de4047bf5415370077cd, NAME => 'TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:13:38,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8eaa77b8f4c3de4047bf5415370077cd, NAME => 'TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8eaa77b8f4c3de4047bf5415370077cd, NAME => 'TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:13:38,858 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35963 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-18T20:13:38,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd. 2024-11-18T20:13:38,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd. as already flushing 2024-11-18T20:13:38,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd. 2024-11-18T20:13:38,858 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 8eaa77b8f4c3de4047bf5415370077cd, NAME => 'TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:13:38,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 8eaa77b8f4c3de4047bf5415370077cd, NAME => 'TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:13:38,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8eaa77b8f4c3de4047bf5415370077cd, NAME => 'TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8eaa77b8f4c3de4047bf5415370077cd, NAME => 'TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:13:39,010 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35963 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-18T20:13:39,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd. 2024-11-18T20:13:39,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd. as already flushing 2024-11-18T20:13:39,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd. 2024-11-18T20:13:39,011 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 8eaa77b8f4c3de4047bf5415370077cd, NAME => 'TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:13:39,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 8eaa77b8f4c3de4047bf5415370077cd, NAME => 'TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:13:39,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8eaa77b8f4c3de4047bf5415370077cd, NAME => 'TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8eaa77b8f4c3de4047bf5415370077cd, NAME => 'TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:13:39,032 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=143 (bloomFilter=true), to=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/.tmp/C/67b001ee8443430dbb148881a5bf85df 2024-11-18T20:13:39,035 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/.tmp/A/9359688205b24faf89fd3d55818f214f as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/A/9359688205b24faf89fd3d55818f214f 2024-11-18T20:13:39,038 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/A/9359688205b24faf89fd3d55818f214f, entries=300, sequenceid=143, filesize=55.7 K 2024-11-18T20:13:39,038 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/.tmp/B/279c46f2a62d4ff4862d72bf147d5229 as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/B/279c46f2a62d4ff4862d72bf147d5229 2024-11-18T20:13:39,041 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/B/279c46f2a62d4ff4862d72bf147d5229, entries=150, sequenceid=143, filesize=11.9 K 2024-11-18T20:13:39,042 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/.tmp/C/67b001ee8443430dbb148881a5bf85df as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/C/67b001ee8443430dbb148881a5bf85df 2024-11-18T20:13:39,045 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/C/67b001ee8443430dbb148881a5bf85df, entries=150, sequenceid=143, filesize=11.9 K 2024-11-18T20:13:39,045 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 8eaa77b8f4c3de4047bf5415370077cd in 1669ms, sequenceid=143, compaction requested=true 2024-11-18T20:13:39,046 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8eaa77b8f4c3de4047bf5415370077cd: 2024-11-18T20:13:39,046 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8eaa77b8f4c3de4047bf5415370077cd:A, priority=-2147483648, current under compaction store size is 1 2024-11-18T20:13:39,046 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:13:39,046 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T20:13:39,046 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8eaa77b8f4c3de4047bf5415370077cd:B, priority=-2147483648, current under compaction store size is 2 2024-11-18T20:13:39,046 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:13:39,046 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T20:13:39,046 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8eaa77b8f4c3de4047bf5415370077cd:C, priority=-2147483648, current under compaction store size is 3 2024-11-18T20:13:39,046 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T20:13:39,046 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 119183 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T20:13:39,046 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36393 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T20:13:39,047 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HStore(1541): 8eaa77b8f4c3de4047bf5415370077cd/A is initiating minor compaction (all files) 2024-11-18T20:13:39,047 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HStore(1541): 8eaa77b8f4c3de4047bf5415370077cd/B is initiating minor compaction (all files) 2024-11-18T20:13:39,047 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8eaa77b8f4c3de4047bf5415370077cd/A in TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd. 2024-11-18T20:13:39,047 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8eaa77b8f4c3de4047bf5415370077cd/B in TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd. 2024-11-18T20:13:39,047 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/A/95ce812f00ed4f3cbbdfb0528d61f7d1, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/A/586b29f729354327b15ca11a6051cc77, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/A/9359688205b24faf89fd3d55818f214f] into tmpdir=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/.tmp, totalSize=116.4 K 2024-11-18T20:13:39,047 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/B/2e3e7c6ea78646e49f09afff72190a85, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/B/8ea784f2d434473fb944939472970ad3, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/B/279c46f2a62d4ff4862d72bf147d5229] into tmpdir=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/.tmp, totalSize=35.5 K 2024-11-18T20:13:39,047 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd. 2024-11-18T20:13:39,047 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd. files: [hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/A/95ce812f00ed4f3cbbdfb0528d61f7d1, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/A/586b29f729354327b15ca11a6051cc77, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/A/9359688205b24faf89fd3d55818f214f] 2024-11-18T20:13:39,047 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] compactions.Compactor(225): Compacting 95ce812f00ed4f3cbbdfb0528d61f7d1, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1731960814908 2024-11-18T20:13:39,047 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.Compactor(225): Compacting 2e3e7c6ea78646e49f09afff72190a85, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1731960814908 2024-11-18T20:13:39,047 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] compactions.Compactor(225): Compacting 586b29f729354327b15ca11a6051cc77, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=129, earliestPutTs=1731960815038 2024-11-18T20:13:39,047 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.Compactor(225): Compacting 8ea784f2d434473fb944939472970ad3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=129, earliestPutTs=1731960815038 2024-11-18T20:13:39,047 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9359688205b24faf89fd3d55818f214f, keycount=300, bloomtype=ROW, size=55.7 K, encoding=NONE, compression=NONE, seqNum=143, earliestPutTs=1731960817301 2024-11-18T20:13:39,048 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.Compactor(225): Compacting 279c46f2a62d4ff4862d72bf147d5229, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=143, earliestPutTs=1731960817349 2024-11-18T20:13:39,054 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8eaa77b8f4c3de4047bf5415370077cd#B#compaction#329 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:13:39,054 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=8eaa77b8f4c3de4047bf5415370077cd] 2024-11-18T20:13:39,055 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/.tmp/B/33b13160ffcb4874ba770969fc043ef2 is 50, key is test_row_0/B:col10/1731960817363/Put/seqid=0 2024-11-18T20:13:39,057 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241118fb13b0e8c2c74a8bbc45fb24342e45ac_8eaa77b8f4c3de4047bf5415370077cd store=[table=TestAcidGuarantees family=A region=8eaa77b8f4c3de4047bf5415370077cd] 2024-11-18T20:13:39,059 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241118fb13b0e8c2c74a8bbc45fb24342e45ac_8eaa77b8f4c3de4047bf5415370077cd, store=[table=TestAcidGuarantees family=A region=8eaa77b8f4c3de4047bf5415370077cd] 2024-11-18T20:13:39,059 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118fb13b0e8c2c74a8bbc45fb24342e45ac_8eaa77b8f4c3de4047bf5415370077cd because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=8eaa77b8f4c3de4047bf5415370077cd] 2024-11-18T20:13:39,065 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742210_1386 (size=12493) 2024-11-18T20:13:39,071 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742211_1387 (size=4469) 2024-11-18T20:13:39,073 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8eaa77b8f4c3de4047bf5415370077cd#A#compaction#330 average throughput is 1.29 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:13:39,074 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/.tmp/A/b378673a59534d59b53e09a3fa6b7c48 is 175, key is test_row_0/A:col10/1731960817363/Put/seqid=0 2024-11-18T20:13:39,078 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742212_1388 (size=31447) 2024-11-18T20:13:39,163 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35963 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-18T20:13:39,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd. 2024-11-18T20:13:39,163 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2902): Flushing 8eaa77b8f4c3de4047bf5415370077cd 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-18T20:13:39,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411183a7dacb0269b4a3182554f48e6dfb16c_8eaa77b8f4c3de4047bf5415370077cd is 50, key is test_row_0/A:col10/1731960817435/Put/seqid=0 2024-11-18T20:13:39,174 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742213_1389 (size=12304) 2024-11-18T20:13:39,469 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/.tmp/B/33b13160ffcb4874ba770969fc043ef2 as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/B/33b13160ffcb4874ba770969fc043ef2 2024-11-18T20:13:39,473 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8eaa77b8f4c3de4047bf5415370077cd/B of 8eaa77b8f4c3de4047bf5415370077cd into 33b13160ffcb4874ba770969fc043ef2(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:13:39,473 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8eaa77b8f4c3de4047bf5415370077cd: 2024-11-18T20:13:39,473 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., storeName=8eaa77b8f4c3de4047bf5415370077cd/B, priority=13, startTime=1731960819046; duration=0sec 2024-11-18T20:13:39,473 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T20:13:39,473 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8eaa77b8f4c3de4047bf5415370077cd:B 2024-11-18T20:13:39,473 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T20:13:39,474 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36393 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T20:13:39,474 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HStore(1541): 8eaa77b8f4c3de4047bf5415370077cd/C is initiating minor compaction (all files) 2024-11-18T20:13:39,474 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8eaa77b8f4c3de4047bf5415370077cd/C in TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd. 2024-11-18T20:13:39,474 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/C/488e247322a34eb9a7675567f725bc26, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/C/130ab60e382345f4afd0d49dfa7d11c0, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/C/67b001ee8443430dbb148881a5bf85df] into tmpdir=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/.tmp, totalSize=35.5 K 2024-11-18T20:13:39,475 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.Compactor(225): Compacting 488e247322a34eb9a7675567f725bc26, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1731960814908 2024-11-18T20:13:39,475 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.Compactor(225): Compacting 130ab60e382345f4afd0d49dfa7d11c0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=129, earliestPutTs=1731960815038 2024-11-18T20:13:39,475 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.Compactor(225): Compacting 67b001ee8443430dbb148881a5bf85df, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=143, earliestPutTs=1731960817349 2024-11-18T20:13:39,481 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8eaa77b8f4c3de4047bf5415370077cd#C#compaction#332 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:13:39,482 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/.tmp/C/184d7265aa78461ba0f8d65d5af1633b is 50, key is test_row_0/C:col10/1731960817363/Put/seqid=0 2024-11-18T20:13:39,482 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/.tmp/A/b378673a59534d59b53e09a3fa6b7c48 as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/A/b378673a59534d59b53e09a3fa6b7c48 2024-11-18T20:13:39,486 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8eaa77b8f4c3de4047bf5415370077cd/A of 8eaa77b8f4c3de4047bf5415370077cd into b378673a59534d59b53e09a3fa6b7c48(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:13:39,486 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8eaa77b8f4c3de4047bf5415370077cd: 2024-11-18T20:13:39,486 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., storeName=8eaa77b8f4c3de4047bf5415370077cd/A, priority=13, startTime=1731960819046; duration=0sec 2024-11-18T20:13:39,486 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:13:39,486 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8eaa77b8f4c3de4047bf5415370077cd:A 2024-11-18T20:13:39,494 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742214_1390 (size=12493) 2024-11-18T20:13:39,499 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/.tmp/C/184d7265aa78461ba0f8d65d5af1633b as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/C/184d7265aa78461ba0f8d65d5af1633b 2024-11-18T20:13:39,503 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8eaa77b8f4c3de4047bf5415370077cd/C of 8eaa77b8f4c3de4047bf5415370077cd into 184d7265aa78461ba0f8d65d5af1633b(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:13:39,503 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8eaa77b8f4c3de4047bf5415370077cd: 2024-11-18T20:13:39,503 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., storeName=8eaa77b8f4c3de4047bf5415370077cd/C, priority=13, startTime=1731960819046; duration=0sec 2024-11-18T20:13:39,503 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:13:39,503 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8eaa77b8f4c3de4047bf5415370077cd:C 2024-11-18T20:13:39,575 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:39,578 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411183a7dacb0269b4a3182554f48e6dfb16c_8eaa77b8f4c3de4047bf5415370077cd to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411183a7dacb0269b4a3182554f48e6dfb16c_8eaa77b8f4c3de4047bf5415370077cd 2024-11-18T20:13:39,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/.tmp/A/a2fd1f59874841f984a78cecf53ebc9c, store: [table=TestAcidGuarantees family=A region=8eaa77b8f4c3de4047bf5415370077cd] 2024-11-18T20:13:39,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/.tmp/A/a2fd1f59874841f984a78cecf53ebc9c is 175, key is test_row_0/A:col10/1731960817435/Put/seqid=0 2024-11-18T20:13:39,583 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742215_1391 (size=31105) 2024-11-18T20:13:39,584 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=168, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/.tmp/A/a2fd1f59874841f984a78cecf53ebc9c 2024-11-18T20:13:39,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/.tmp/B/2514fc1351684a2d8e4ac89d24247572 is 50, key is test_row_0/B:col10/1731960817435/Put/seqid=0 2024-11-18T20:13:39,597 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742216_1392 (size=12151) 2024-11-18T20:13:39,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(8855): Flush requested on 8eaa77b8f4c3de4047bf5415370077cd 2024-11-18T20:13:39,618 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd. as already flushing 2024-11-18T20:13:39,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-18T20:13:39,640 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:39,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39882 deadline: 1731960879634, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:39,641 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:39,641 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:39,641 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:39,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39866 deadline: 1731960879634, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:39,641 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:13:39,641 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:39,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39924 deadline: 1731960879638, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:39,642 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:39,642 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:39,642 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:13:39,642 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to c0a89b2656d4,35963,1731960718223 for 'test_row_0' in 8eaa77b8f4c3de4047bf5415370077cd of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2195 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-18T20:13:39,642 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:39,642 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:39,642 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:13:39,746 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:39,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39882 deadline: 1731960879743, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:39,746 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:39,746 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:39,747 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:13:39,759 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:39,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39924 deadline: 1731960879753, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:39,759 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:39,759 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:39,759 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:13:39,956 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:39,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39882 deadline: 1731960879953, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:39,956 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:39,956 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:39,956 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:13:39,968 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:39,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39924 deadline: 1731960879963, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:39,969 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:39,969 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:39,969 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:13:39,998 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=168 (bloomFilter=true), to=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/.tmp/B/2514fc1351684a2d8e4ac89d24247572 2024-11-18T20:13:40,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/.tmp/C/bd4b35b2165e4c7c8b671a04fe08f5da is 50, key is test_row_0/C:col10/1731960817435/Put/seqid=0 2024-11-18T20:13:40,007 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742217_1393 (size=12151) 2024-11-18T20:13:40,008 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=168 (bloomFilter=true), to=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/.tmp/C/bd4b35b2165e4c7c8b671a04fe08f5da 2024-11-18T20:13:40,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/.tmp/A/a2fd1f59874841f984a78cecf53ebc9c as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/A/a2fd1f59874841f984a78cecf53ebc9c 2024-11-18T20:13:40,016 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/A/a2fd1f59874841f984a78cecf53ebc9c, entries=150, sequenceid=168, filesize=30.4 K 2024-11-18T20:13:40,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/.tmp/B/2514fc1351684a2d8e4ac89d24247572 as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/B/2514fc1351684a2d8e4ac89d24247572 2024-11-18T20:13:40,020 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/B/2514fc1351684a2d8e4ac89d24247572, entries=150, sequenceid=168, filesize=11.9 K 2024-11-18T20:13:40,020 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/.tmp/C/bd4b35b2165e4c7c8b671a04fe08f5da as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/C/bd4b35b2165e4c7c8b671a04fe08f5da 2024-11-18T20:13:40,023 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/C/bd4b35b2165e4c7c8b671a04fe08f5da, entries=150, sequenceid=168, filesize=11.9 K 2024-11-18T20:13:40,024 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 8eaa77b8f4c3de4047bf5415370077cd in 861ms, sequenceid=168, compaction requested=false 2024-11-18T20:13:40,024 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2603): Flush status journal for 8eaa77b8f4c3de4047bf5415370077cd: 2024-11-18T20:13:40,024 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd. 2024-11-18T20:13:40,024 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=106 2024-11-18T20:13:40,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.HMaster(4169): Remote procedure done, pid=106 2024-11-18T20:13:40,027 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=106, resume processing ppid=105 2024-11-18T20:13:40,027 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=106, ppid=105, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.5400 sec 2024-11-18T20:13:40,028 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=105, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees in 2.5440 sec 2024-11-18T20:13:40,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(8855): Flush requested on 8eaa77b8f4c3de4047bf5415370077cd 2024-11-18T20:13:40,266 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-18T20:13:40,266 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8eaa77b8f4c3de4047bf5415370077cd 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-18T20:13:40,273 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111833db8552e4124ed695a73efe62c6b815_8eaa77b8f4c3de4047bf5415370077cd is 50, key is test_row_0/A:col10/1731960819620/Put/seqid=0 2024-11-18T20:13:40,280 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742218_1394 (size=14794) 2024-11-18T20:13:40,283 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:40,293 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111833db8552e4124ed695a73efe62c6b815_8eaa77b8f4c3de4047bf5415370077cd to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111833db8552e4124ed695a73efe62c6b815_8eaa77b8f4c3de4047bf5415370077cd 2024-11-18T20:13:40,293 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/.tmp/A/1cefe7769199475883e4e4971a63f26c, store: [table=TestAcidGuarantees family=A region=8eaa77b8f4c3de4047bf5415370077cd] 2024-11-18T20:13:40,294 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/.tmp/A/1cefe7769199475883e4e4971a63f26c is 175, key is test_row_0/A:col10/1731960819620/Put/seqid=0 2024-11-18T20:13:40,297 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742219_1395 (size=39749) 2024-11-18T20:13:40,354 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:40,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39882 deadline: 1731960880348, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:40,355 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:40,355 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:40,355 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:13:40,359 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:40,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39924 deadline: 1731960880355, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:40,360 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:40,360 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:40,360 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:13:40,469 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:40,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39882 deadline: 1731960880463, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:40,469 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:40,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39924 deadline: 1731960880463, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:40,469 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:40,469 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:40,469 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:13:40,469 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:40,470 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:40,470 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:13:40,679 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:40,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39882 deadline: 1731960880673, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:40,680 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:40,680 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:40,680 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:40,680 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:13:40,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39924 deadline: 1731960880673, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:40,680 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:40,680 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:40,680 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:13:40,698 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=183, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/.tmp/A/1cefe7769199475883e4e4971a63f26c 2024-11-18T20:13:40,704 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/.tmp/B/2cce7795a653443baa5ed2a7a2266e09 is 50, key is test_row_0/B:col10/1731960819620/Put/seqid=0 2024-11-18T20:13:40,707 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742220_1396 (size=12151) 2024-11-18T20:13:40,990 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:40,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39882 deadline: 1731960880983, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:40,991 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:40,991 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:40,991 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:13:41,001 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:41,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39924 deadline: 1731960880993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:41,001 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:41,001 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:41,001 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:13:41,108 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=183 (bloomFilter=true), to=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/.tmp/B/2cce7795a653443baa5ed2a7a2266e09 2024-11-18T20:13:41,114 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/.tmp/C/32f4a989f1f9479ea7d0300d4fae626a is 50, key is test_row_0/C:col10/1731960819620/Put/seqid=0 2024-11-18T20:13:41,117 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742221_1397 (size=12151) 2024-11-18T20:13:41,509 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:41,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39882 deadline: 1731960881503, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:41,509 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:41,509 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:41,509 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:13:41,517 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=183 (bloomFilter=true), to=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/.tmp/C/32f4a989f1f9479ea7d0300d4fae626a 2024-11-18T20:13:41,517 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:41,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39924 deadline: 1731960881513, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:41,518 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:41,518 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:41,518 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:13:41,521 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/.tmp/A/1cefe7769199475883e4e4971a63f26c as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/A/1cefe7769199475883e4e4971a63f26c 2024-11-18T20:13:41,524 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/A/1cefe7769199475883e4e4971a63f26c, entries=200, sequenceid=183, filesize=38.8 K 2024-11-18T20:13:41,524 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/.tmp/B/2cce7795a653443baa5ed2a7a2266e09 as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/B/2cce7795a653443baa5ed2a7a2266e09 2024-11-18T20:13:41,527 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/B/2cce7795a653443baa5ed2a7a2266e09, entries=150, sequenceid=183, filesize=11.9 K 2024-11-18T20:13:41,528 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/.tmp/C/32f4a989f1f9479ea7d0300d4fae626a as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/C/32f4a989f1f9479ea7d0300d4fae626a 2024-11-18T20:13:41,531 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/C/32f4a989f1f9479ea7d0300d4fae626a, entries=150, sequenceid=183, filesize=11.9 K 2024-11-18T20:13:41,532 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 8eaa77b8f4c3de4047bf5415370077cd in 1266ms, sequenceid=183, compaction requested=true 2024-11-18T20:13:41,532 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8eaa77b8f4c3de4047bf5415370077cd: 2024-11-18T20:13:41,532 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8eaa77b8f4c3de4047bf5415370077cd:A, priority=-2147483648, current under compaction store size is 1 2024-11-18T20:13:41,532 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:13:41,532 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8eaa77b8f4c3de4047bf5415370077cd:B, priority=-2147483648, current under compaction store size is 2 2024-11-18T20:13:41,532 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T20:13:41,532 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:13:41,532 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T20:13:41,532 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8eaa77b8f4c3de4047bf5415370077cd:C, priority=-2147483648, current under compaction store size is 3 2024-11-18T20:13:41,532 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T20:13:41,533 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T20:13:41,533 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102301 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T20:13:41,533 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HStore(1541): 8eaa77b8f4c3de4047bf5415370077cd/B is initiating minor compaction (all files) 2024-11-18T20:13:41,533 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HStore(1541): 8eaa77b8f4c3de4047bf5415370077cd/A is initiating minor compaction (all files) 2024-11-18T20:13:41,533 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8eaa77b8f4c3de4047bf5415370077cd/B in TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd. 2024-11-18T20:13:41,533 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8eaa77b8f4c3de4047bf5415370077cd/A in TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd. 2024-11-18T20:13:41,533 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/B/33b13160ffcb4874ba770969fc043ef2, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/B/2514fc1351684a2d8e4ac89d24247572, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/B/2cce7795a653443baa5ed2a7a2266e09] into tmpdir=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/.tmp, totalSize=35.9 K 2024-11-18T20:13:41,533 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/A/b378673a59534d59b53e09a3fa6b7c48, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/A/a2fd1f59874841f984a78cecf53ebc9c, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/A/1cefe7769199475883e4e4971a63f26c] into tmpdir=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/.tmp, totalSize=99.9 K 2024-11-18T20:13:41,533 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd. 2024-11-18T20:13:41,533 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd. files: [hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/A/b378673a59534d59b53e09a3fa6b7c48, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/A/a2fd1f59874841f984a78cecf53ebc9c, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/A/1cefe7769199475883e4e4971a63f26c] 2024-11-18T20:13:41,533 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.Compactor(225): Compacting 33b13160ffcb4874ba770969fc043ef2, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=143, earliestPutTs=1731960817349 2024-11-18T20:13:41,533 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] compactions.Compactor(225): Compacting b378673a59534d59b53e09a3fa6b7c48, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=143, earliestPutTs=1731960817349 2024-11-18T20:13:41,534 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.Compactor(225): Compacting 2514fc1351684a2d8e4ac89d24247572, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=168, earliestPutTs=1731960817425 2024-11-18T20:13:41,534 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] compactions.Compactor(225): Compacting a2fd1f59874841f984a78cecf53ebc9c, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=168, earliestPutTs=1731960817425 2024-11-18T20:13:41,534 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1cefe7769199475883e4e4971a63f26c, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=183, earliestPutTs=1731960819620 2024-11-18T20:13:41,534 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.Compactor(225): Compacting 2cce7795a653443baa5ed2a7a2266e09, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=183, earliestPutTs=1731960819620 2024-11-18T20:13:41,538 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=8eaa77b8f4c3de4047bf5415370077cd] 2024-11-18T20:13:41,539 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8eaa77b8f4c3de4047bf5415370077cd#B#compaction#338 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:13:41,540 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/.tmp/B/2c5c4e80bf194e99a8386b7b5e720ba5 is 50, key is test_row_0/B:col10/1731960819620/Put/seqid=0 2024-11-18T20:13:41,542 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241118fd5fdba95d5c4dcebc60d3f19069e426_8eaa77b8f4c3de4047bf5415370077cd store=[table=TestAcidGuarantees family=A region=8eaa77b8f4c3de4047bf5415370077cd] 2024-11-18T20:13:41,544 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241118fd5fdba95d5c4dcebc60d3f19069e426_8eaa77b8f4c3de4047bf5415370077cd, store=[table=TestAcidGuarantees family=A region=8eaa77b8f4c3de4047bf5415370077cd] 2024-11-18T20:13:41,544 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118fd5fdba95d5c4dcebc60d3f19069e426_8eaa77b8f4c3de4047bf5415370077cd because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=8eaa77b8f4c3de4047bf5415370077cd] 2024-11-18T20:13:41,552 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742222_1398 (size=12595) 2024-11-18T20:13:41,553 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742223_1399 (size=4469) 2024-11-18T20:13:41,554 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8eaa77b8f4c3de4047bf5415370077cd#A#compaction#339 average throughput is 1.63 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:13:41,555 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/.tmp/A/d300b651488647d6bf3c7662bb435e6d is 175, key is test_row_0/A:col10/1731960819620/Put/seqid=0 2024-11-18T20:13:41,559 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742224_1400 (size=31549) 2024-11-18T20:13:41,564 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/.tmp/A/d300b651488647d6bf3c7662bb435e6d as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/A/d300b651488647d6bf3c7662bb435e6d 2024-11-18T20:13:41,569 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8eaa77b8f4c3de4047bf5415370077cd/A of 8eaa77b8f4c3de4047bf5415370077cd into d300b651488647d6bf3c7662bb435e6d(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:13:41,569 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8eaa77b8f4c3de4047bf5415370077cd: 2024-11-18T20:13:41,569 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., storeName=8eaa77b8f4c3de4047bf5415370077cd/A, priority=13, startTime=1731960821532; duration=0sec 2024-11-18T20:13:41,569 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T20:13:41,569 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8eaa77b8f4c3de4047bf5415370077cd:A 2024-11-18T20:13:41,569 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T20:13:41,570 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T20:13:41,570 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HStore(1541): 8eaa77b8f4c3de4047bf5415370077cd/C is initiating minor compaction (all files) 2024-11-18T20:13:41,570 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8eaa77b8f4c3de4047bf5415370077cd/C in TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd. 2024-11-18T20:13:41,570 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/C/184d7265aa78461ba0f8d65d5af1633b, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/C/bd4b35b2165e4c7c8b671a04fe08f5da, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/C/32f4a989f1f9479ea7d0300d4fae626a] into tmpdir=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/.tmp, totalSize=35.9 K 2024-11-18T20:13:41,571 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] compactions.Compactor(225): Compacting 184d7265aa78461ba0f8d65d5af1633b, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=143, earliestPutTs=1731960817349 2024-11-18T20:13:41,571 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] compactions.Compactor(225): Compacting bd4b35b2165e4c7c8b671a04fe08f5da, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=168, earliestPutTs=1731960817425 2024-11-18T20:13:41,571 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] compactions.Compactor(225): Compacting 32f4a989f1f9479ea7d0300d4fae626a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=183, earliestPutTs=1731960819620 2024-11-18T20:13:41,577 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8eaa77b8f4c3de4047bf5415370077cd#C#compaction#340 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:13:41,577 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/.tmp/C/941c6ba825684c128518e0753f084de3 is 50, key is test_row_0/C:col10/1731960819620/Put/seqid=0 2024-11-18T20:13:41,581 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742225_1401 (size=12595) 2024-11-18T20:13:41,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-18T20:13:41,643 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-18T20:13:41,643 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-18T20:13:41,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] procedure2.ProcedureExecutor(1139): Stored pid=107, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees 2024-11-18T20:13:41,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-18T20:13:41,645 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=107, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-18T20:13:41,646 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=107, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-18T20:13:41,646 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=108, ppid=107, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-18T20:13:41,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(8855): Flush requested on 8eaa77b8f4c3de4047bf5415370077cd 2024-11-18T20:13:41,670 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-18T20:13:41,670 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8eaa77b8f4c3de4047bf5415370077cd 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-18T20:13:41,676 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118193793844f7a4b9fb5e484d2ce5540f6_8eaa77b8f4c3de4047bf5415370077cd is 50, key is test_row_0/A:col10/1731960821669/Put/seqid=0 2024-11-18T20:13:41,679 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742226_1402 (size=14794) 2024-11-18T20:13:41,679 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:41,682 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118193793844f7a4b9fb5e484d2ce5540f6_8eaa77b8f4c3de4047bf5415370077cd to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118193793844f7a4b9fb5e484d2ce5540f6_8eaa77b8f4c3de4047bf5415370077cd 2024-11-18T20:13:41,683 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/.tmp/A/6eb81deeae224aa496c3e80fd807719c, store: [table=TestAcidGuarantees family=A region=8eaa77b8f4c3de4047bf5415370077cd] 2024-11-18T20:13:41,684 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/.tmp/A/6eb81deeae224aa496c3e80fd807719c is 175, key is test_row_0/A:col10/1731960821669/Put/seqid=0 2024-11-18T20:13:41,687 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742227_1403 (size=39749) 2024-11-18T20:13:41,746 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:41,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39866 deadline: 1731960881739, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:41,746 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:41,746 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:41,746 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:13:41,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-18T20:13:41,798 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35963 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-18T20:13:41,798 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd. 2024-11-18T20:13:41,798 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd. as already flushing 2024-11-18T20:13:41,798 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd. 2024-11-18T20:13:41,798 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 8eaa77b8f4c3de4047bf5415370077cd, NAME => 'TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:13:41,798 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 8eaa77b8f4c3de4047bf5415370077cd, NAME => 'TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:13:41,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8eaa77b8f4c3de4047bf5415370077cd, NAME => 'TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8eaa77b8f4c3de4047bf5415370077cd, NAME => 'TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:13:41,861 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:41,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39866 deadline: 1731960881853, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:41,862 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:41,862 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:41,862 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:13:41,950 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35963 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-18T20:13:41,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd. 2024-11-18T20:13:41,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd. as already flushing 2024-11-18T20:13:41,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd. 2024-11-18T20:13:41,951 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 8eaa77b8f4c3de4047bf5415370077cd, NAME => 'TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:13:41,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 8eaa77b8f4c3de4047bf5415370077cd, NAME => 'TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:13:41,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8eaa77b8f4c3de4047bf5415370077cd, NAME => 'TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8eaa77b8f4c3de4047bf5415370077cd, NAME => 'TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:13:41,956 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/.tmp/B/2c5c4e80bf194e99a8386b7b5e720ba5 as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/B/2c5c4e80bf194e99a8386b7b5e720ba5 2024-11-18T20:13:41,960 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8eaa77b8f4c3de4047bf5415370077cd/B of 8eaa77b8f4c3de4047bf5415370077cd into 2c5c4e80bf194e99a8386b7b5e720ba5(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:13:41,960 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8eaa77b8f4c3de4047bf5415370077cd: 2024-11-18T20:13:41,960 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., storeName=8eaa77b8f4c3de4047bf5415370077cd/B, priority=13, startTime=1731960821532; duration=0sec 2024-11-18T20:13:41,961 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:13:41,961 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8eaa77b8f4c3de4047bf5415370077cd:B 2024-11-18T20:13:41,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-18T20:13:41,985 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/.tmp/C/941c6ba825684c128518e0753f084de3 as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/C/941c6ba825684c128518e0753f084de3 2024-11-18T20:13:41,989 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8eaa77b8f4c3de4047bf5415370077cd/C of 8eaa77b8f4c3de4047bf5415370077cd into 941c6ba825684c128518e0753f084de3(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:13:41,989 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8eaa77b8f4c3de4047bf5415370077cd: 2024-11-18T20:13:41,989 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., storeName=8eaa77b8f4c3de4047bf5415370077cd/C, priority=13, startTime=1731960821532; duration=0sec 2024-11-18T20:13:41,989 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:13:41,989 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8eaa77b8f4c3de4047bf5415370077cd:C 2024-11-18T20:13:42,066 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:42,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39866 deadline: 1731960882063, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:42,067 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:42,067 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:42,067 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:13:42,088 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=209, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/.tmp/A/6eb81deeae224aa496c3e80fd807719c 2024-11-18T20:13:42,095 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/.tmp/B/44f83273953d48f7b1d2dc70fa334c21 is 50, key is test_row_0/B:col10/1731960821669/Put/seqid=0 2024-11-18T20:13:42,101 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742228_1404 (size=12151) 2024-11-18T20:13:42,103 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35963 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-18T20:13:42,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd. 2024-11-18T20:13:42,104 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd. as already flushing 2024-11-18T20:13:42,104 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd. 2024-11-18T20:13:42,104 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 8eaa77b8f4c3de4047bf5415370077cd, NAME => 'TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:13:42,104 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 8eaa77b8f4c3de4047bf5415370077cd, NAME => 'TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:13:42,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8eaa77b8f4c3de4047bf5415370077cd, NAME => 'TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8eaa77b8f4c3de4047bf5415370077cd, NAME => 'TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:13:42,256 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35963 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-18T20:13:42,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd. 2024-11-18T20:13:42,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd. as already flushing 2024-11-18T20:13:42,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd. 2024-11-18T20:13:42,256 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 8eaa77b8f4c3de4047bf5415370077cd, NAME => 'TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:13:42,257 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 8eaa77b8f4c3de4047bf5415370077cd, NAME => 'TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:13:42,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8eaa77b8f4c3de4047bf5415370077cd, NAME => 'TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8eaa77b8f4c3de4047bf5415370077cd, NAME => 'TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:13:42,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-18T20:13:42,379 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:42,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39866 deadline: 1731960882373, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:42,379 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:42,379 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:42,379 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:13:42,409 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35963 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-18T20:13:42,409 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd. 2024-11-18T20:13:42,409 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd. as already flushing 2024-11-18T20:13:42,409 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd. 2024-11-18T20:13:42,409 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 8eaa77b8f4c3de4047bf5415370077cd, NAME => 'TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:13:42,409 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 8eaa77b8f4c3de4047bf5415370077cd, NAME => 'TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:13:42,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8eaa77b8f4c3de4047bf5415370077cd, NAME => 'TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8eaa77b8f4c3de4047bf5415370077cd, NAME => 'TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:13:42,502 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=209 (bloomFilter=true), to=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/.tmp/B/44f83273953d48f7b1d2dc70fa334c21 2024-11-18T20:13:42,507 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/.tmp/C/c2150536f53a48afb12ba0bbcef65e30 is 50, key is test_row_0/C:col10/1731960821669/Put/seqid=0 2024-11-18T20:13:42,510 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742229_1405 (size=12151) 2024-11-18T20:13:42,511 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=209 (bloomFilter=true), to=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/.tmp/C/c2150536f53a48afb12ba0bbcef65e30 2024-11-18T20:13:42,513 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/.tmp/A/6eb81deeae224aa496c3e80fd807719c as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/A/6eb81deeae224aa496c3e80fd807719c 2024-11-18T20:13:42,516 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/A/6eb81deeae224aa496c3e80fd807719c, entries=200, sequenceid=209, filesize=38.8 K 2024-11-18T20:13:42,519 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/.tmp/B/44f83273953d48f7b1d2dc70fa334c21 as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/B/44f83273953d48f7b1d2dc70fa334c21 2024-11-18T20:13:42,522 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/B/44f83273953d48f7b1d2dc70fa334c21, entries=150, sequenceid=209, filesize=11.9 K 2024-11-18T20:13:42,522 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/.tmp/C/c2150536f53a48afb12ba0bbcef65e30 as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/C/c2150536f53a48afb12ba0bbcef65e30 2024-11-18T20:13:42,526 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/C/c2150536f53a48afb12ba0bbcef65e30, entries=150, sequenceid=209, filesize=11.9 K 2024-11-18T20:13:42,529 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 8eaa77b8f4c3de4047bf5415370077cd in 859ms, sequenceid=209, compaction requested=false 2024-11-18T20:13:42,529 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8eaa77b8f4c3de4047bf5415370077cd: 2024-11-18T20:13:42,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(8855): Flush requested on 8eaa77b8f4c3de4047bf5415370077cd 2024-11-18T20:13:42,530 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-18T20:13:42,530 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8eaa77b8f4c3de4047bf5415370077cd 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-18T20:13:42,540 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118ea198b77737e422abb6b17be84f80a94_8eaa77b8f4c3de4047bf5415370077cd is 50, key is test_row_0/A:col10/1731960821721/Put/seqid=0 2024-11-18T20:13:42,543 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742230_1406 (size=14794) 2024-11-18T20:13:42,561 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35963 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-18T20:13:42,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd. 2024-11-18T20:13:42,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd. as already flushing 2024-11-18T20:13:42,562 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd. 2024-11-18T20:13:42,562 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 8eaa77b8f4c3de4047bf5415370077cd, NAME => 'TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:13:42,562 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 8eaa77b8f4c3de4047bf5415370077cd, NAME => 'TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:13:42,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8eaa77b8f4c3de4047bf5415370077cd, NAME => 'TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8eaa77b8f4c3de4047bf5415370077cd, NAME => 'TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:13:42,605 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:42,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39924 deadline: 1731960882599, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:42,606 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:42,606 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:42,606 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:13:42,607 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:42,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39882 deadline: 1731960882605, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:42,607 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:42,608 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:42,608 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:13:42,714 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35963 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-18T20:13:42,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd. 2024-11-18T20:13:42,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd. as already flushing 2024-11-18T20:13:42,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd. 2024-11-18T20:13:42,714 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 8eaa77b8f4c3de4047bf5415370077cd, NAME => 'TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:13:42,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 8eaa77b8f4c3de4047bf5415370077cd, NAME => 'TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:13:42,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8eaa77b8f4c3de4047bf5415370077cd, NAME => 'TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8eaa77b8f4c3de4047bf5415370077cd, NAME => 'TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:13:42,720 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:42,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39924 deadline: 1731960882713, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:42,720 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:42,720 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:42,720 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:42,720 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:13:42,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39882 deadline: 1731960882713, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:42,720 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:42,720 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:42,721 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:13:42,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-18T20:13:42,867 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35963 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-18T20:13:42,867 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd. 2024-11-18T20:13:42,867 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd. as already flushing 2024-11-18T20:13:42,867 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd. 2024-11-18T20:13:42,867 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 8eaa77b8f4c3de4047bf5415370077cd, NAME => 'TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:13:42,867 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 8eaa77b8f4c3de4047bf5415370077cd, NAME => 'TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:13:42,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8eaa77b8f4c3de4047bf5415370077cd, NAME => 'TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8eaa77b8f4c3de4047bf5415370077cd, NAME => 'TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:13:42,892 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:42,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39866 deadline: 1731960882884, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:42,892 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:42,893 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:42,893 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:13:42,929 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:42,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39924 deadline: 1731960882923, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:42,930 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:42,930 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:42,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39882 deadline: 1731960882923, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:42,930 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:42,930 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:13:42,930 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:42,930 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:42,930 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:13:42,944 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:42,947 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118ea198b77737e422abb6b17be84f80a94_8eaa77b8f4c3de4047bf5415370077cd to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118ea198b77737e422abb6b17be84f80a94_8eaa77b8f4c3de4047bf5415370077cd 2024-11-18T20:13:42,947 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/.tmp/A/e53f36ebfc4c42ecac30d3885f453e36, store: [table=TestAcidGuarantees family=A region=8eaa77b8f4c3de4047bf5415370077cd] 2024-11-18T20:13:42,948 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/.tmp/A/e53f36ebfc4c42ecac30d3885f453e36 is 175, key is test_row_0/A:col10/1731960821721/Put/seqid=0 2024-11-18T20:13:42,951 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742231_1407 (size=39749) 2024-11-18T20:13:43,019 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35963 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-18T20:13:43,019 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd. 2024-11-18T20:13:43,019 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd. as already flushing 2024-11-18T20:13:43,019 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd. 2024-11-18T20:13:43,020 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 8eaa77b8f4c3de4047bf5415370077cd, NAME => 'TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:13:43,020 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 8eaa77b8f4c3de4047bf5415370077cd, NAME => 'TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:13:43,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8eaa77b8f4c3de4047bf5415370077cd, NAME => 'TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8eaa77b8f4c3de4047bf5415370077cd, NAME => 'TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:13:43,172 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35963 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-18T20:13:43,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd. 2024-11-18T20:13:43,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd. as already flushing 2024-11-18T20:13:43,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd. 2024-11-18T20:13:43,172 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 8eaa77b8f4c3de4047bf5415370077cd, NAME => 'TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:13:43,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 8eaa77b8f4c3de4047bf5415370077cd, NAME => 'TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:13:43,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8eaa77b8f4c3de4047bf5415370077cd, NAME => 'TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8eaa77b8f4c3de4047bf5415370077cd, NAME => 'TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:13:43,235 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:43,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39924 deadline: 1731960883233, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:43,235 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:43,235 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:43,235 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:13:43,244 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:43,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39882 deadline: 1731960883243, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:43,244 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:43,244 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:43,244 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:13:43,324 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35963 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-18T20:13:43,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd. 2024-11-18T20:13:43,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd. as already flushing 2024-11-18T20:13:43,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd. 2024-11-18T20:13:43,325 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 8eaa77b8f4c3de4047bf5415370077cd, NAME => 'TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:13:43,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 8eaa77b8f4c3de4047bf5415370077cd, NAME => 'TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:13:43,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8eaa77b8f4c3de4047bf5415370077cd, NAME => 'TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8eaa77b8f4c3de4047bf5415370077cd, NAME => 'TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:13:43,352 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=223, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/.tmp/A/e53f36ebfc4c42ecac30d3885f453e36 2024-11-18T20:13:43,358 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/.tmp/B/b96d30aaaee34e3f8b4c79d610182434 is 50, key is test_row_0/B:col10/1731960821721/Put/seqid=0 2024-11-18T20:13:43,360 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742232_1408 (size=12151) 2024-11-18T20:13:43,477 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35963 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-18T20:13:43,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd. 2024-11-18T20:13:43,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd. as already flushing 2024-11-18T20:13:43,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd. 2024-11-18T20:13:43,477 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 8eaa77b8f4c3de4047bf5415370077cd, NAME => 'TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:13:43,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 8eaa77b8f4c3de4047bf5415370077cd, NAME => 'TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:13:43,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8eaa77b8f4c3de4047bf5415370077cd, NAME => 'TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8eaa77b8f4c3de4047bf5415370077cd, NAME => 'TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:13:43,630 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35963 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-18T20:13:43,630 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd. 2024-11-18T20:13:43,630 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd. as already flushing 2024-11-18T20:13:43,630 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd. 2024-11-18T20:13:43,630 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 8eaa77b8f4c3de4047bf5415370077cd, NAME => 'TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:13:43,630 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 8eaa77b8f4c3de4047bf5415370077cd, NAME => 'TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:13:43,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8eaa77b8f4c3de4047bf5415370077cd, NAME => 'TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8eaa77b8f4c3de4047bf5415370077cd, NAME => 'TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:13:43,745 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:43,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39924 deadline: 1731960883743, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:43,746 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:43,746 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:43,746 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:13:43,759 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:43,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39882 deadline: 1731960883753, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:43,759 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:43,759 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:43,760 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:13:43,761 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=223 (bloomFilter=true), to=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/.tmp/B/b96d30aaaee34e3f8b4c79d610182434 2024-11-18T20:13:43,767 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/.tmp/C/3e4de34e127544cba9c7b19533d00c3f is 50, key is test_row_0/C:col10/1731960821721/Put/seqid=0 2024-11-18T20:13:43,769 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742233_1409 (size=12151) 2024-11-18T20:13:43,782 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35963 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-18T20:13:43,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd. 2024-11-18T20:13:43,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd. as already flushing 2024-11-18T20:13:43,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd. 2024-11-18T20:13:43,783 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 8eaa77b8f4c3de4047bf5415370077cd, NAME => 'TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:13:43,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 8eaa77b8f4c3de4047bf5415370077cd, NAME => 'TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:13:43,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8eaa77b8f4c3de4047bf5415370077cd, NAME => 'TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8eaa77b8f4c3de4047bf5415370077cd, NAME => 'TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:13:43,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-18T20:13:43,911 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:43,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39866 deadline: 1731960883903, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:43,911 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to c0a89b2656d4,35963,1731960718223 for 'test_row_2' in 8eaa77b8f4c3de4047bf5415370077cd of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2172 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-18T20:13:43,912 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:43,912 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:43,912 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:13:43,935 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35963 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-18T20:13:43,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd. 2024-11-18T20:13:43,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd. as already flushing 2024-11-18T20:13:43,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd. 2024-11-18T20:13:43,935 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 8eaa77b8f4c3de4047bf5415370077cd, NAME => 'TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:13:43,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 8eaa77b8f4c3de4047bf5415370077cd, NAME => 'TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:13:43,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8eaa77b8f4c3de4047bf5415370077cd, NAME => 'TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8eaa77b8f4c3de4047bf5415370077cd, NAME => 'TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:13:44,088 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35963 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-18T20:13:44,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd. 2024-11-18T20:13:44,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd. as already flushing 2024-11-18T20:13:44,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd. 2024-11-18T20:13:44,088 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 8eaa77b8f4c3de4047bf5415370077cd, NAME => 'TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:13:44,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 8eaa77b8f4c3de4047bf5415370077cd, NAME => 'TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:13:44,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8eaa77b8f4c3de4047bf5415370077cd, NAME => 'TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8eaa77b8f4c3de4047bf5415370077cd, NAME => 'TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:13:44,170 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=223 (bloomFilter=true), to=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/.tmp/C/3e4de34e127544cba9c7b19533d00c3f 2024-11-18T20:13:44,174 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/.tmp/A/e53f36ebfc4c42ecac30d3885f453e36 as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/A/e53f36ebfc4c42ecac30d3885f453e36 2024-11-18T20:13:44,177 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/A/e53f36ebfc4c42ecac30d3885f453e36, entries=200, sequenceid=223, filesize=38.8 K 2024-11-18T20:13:44,179 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/.tmp/B/b96d30aaaee34e3f8b4c79d610182434 as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/B/b96d30aaaee34e3f8b4c79d610182434 2024-11-18T20:13:44,182 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/B/b96d30aaaee34e3f8b4c79d610182434, entries=150, sequenceid=223, filesize=11.9 K 2024-11-18T20:13:44,183 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/.tmp/C/3e4de34e127544cba9c7b19533d00c3f as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/C/3e4de34e127544cba9c7b19533d00c3f 2024-11-18T20:13:44,185 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/C/3e4de34e127544cba9c7b19533d00c3f, entries=150, sequenceid=223, filesize=11.9 K 2024-11-18T20:13:44,186 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 8eaa77b8f4c3de4047bf5415370077cd in 1656ms, sequenceid=223, compaction requested=true 2024-11-18T20:13:44,186 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8eaa77b8f4c3de4047bf5415370077cd: 2024-11-18T20:13:44,186 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8eaa77b8f4c3de4047bf5415370077cd:A, priority=-2147483648, current under compaction store size is 1 2024-11-18T20:13:44,186 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:13:44,186 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T20:13:44,187 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8eaa77b8f4c3de4047bf5415370077cd:B, priority=-2147483648, current under compaction store size is 2 2024-11-18T20:13:44,187 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:13:44,187 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T20:13:44,187 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8eaa77b8f4c3de4047bf5415370077cd:C, priority=-2147483648, current under compaction store size is 3 2024-11-18T20:13:44,187 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T20:13:44,187 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T20:13:44,187 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 111047 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T20:13:44,187 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HStore(1541): 8eaa77b8f4c3de4047bf5415370077cd/B is initiating minor compaction (all files) 2024-11-18T20:13:44,187 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HStore(1541): 8eaa77b8f4c3de4047bf5415370077cd/A is initiating minor compaction (all files) 2024-11-18T20:13:44,187 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8eaa77b8f4c3de4047bf5415370077cd/A in TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd. 2024-11-18T20:13:44,187 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8eaa77b8f4c3de4047bf5415370077cd/B in TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd. 2024-11-18T20:13:44,188 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/B/2c5c4e80bf194e99a8386b7b5e720ba5, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/B/44f83273953d48f7b1d2dc70fa334c21, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/B/b96d30aaaee34e3f8b4c79d610182434] into tmpdir=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/.tmp, totalSize=36.0 K 2024-11-18T20:13:44,188 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/A/d300b651488647d6bf3c7662bb435e6d, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/A/6eb81deeae224aa496c3e80fd807719c, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/A/e53f36ebfc4c42ecac30d3885f453e36] into tmpdir=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/.tmp, totalSize=108.4 K 2024-11-18T20:13:44,188 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd. 2024-11-18T20:13:44,188 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd. files: [hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/A/d300b651488647d6bf3c7662bb435e6d, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/A/6eb81deeae224aa496c3e80fd807719c, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/A/e53f36ebfc4c42ecac30d3885f453e36] 2024-11-18T20:13:44,188 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2c5c4e80bf194e99a8386b7b5e720ba5, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=183, earliestPutTs=1731960819620 2024-11-18T20:13:44,188 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.Compactor(225): Compacting d300b651488647d6bf3c7662bb435e6d, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=183, earliestPutTs=1731960819620 2024-11-18T20:13:44,188 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] compactions.Compactor(225): Compacting 44f83273953d48f7b1d2dc70fa334c21, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=209, earliestPutTs=1731960820343 2024-11-18T20:13:44,188 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.Compactor(225): Compacting 6eb81deeae224aa496c3e80fd807719c, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=209, earliestPutTs=1731960820343 2024-11-18T20:13:44,188 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] compactions.Compactor(225): Compacting b96d30aaaee34e3f8b4c79d610182434, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=223, earliestPutTs=1731960821721 2024-11-18T20:13:44,189 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.Compactor(225): Compacting e53f36ebfc4c42ecac30d3885f453e36, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=223, earliestPutTs=1731960821714 2024-11-18T20:13:44,193 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=8eaa77b8f4c3de4047bf5415370077cd] 2024-11-18T20:13:44,198 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411187f128565ff8d4f5ca38f0981e6df026e_8eaa77b8f4c3de4047bf5415370077cd store=[table=TestAcidGuarantees family=A region=8eaa77b8f4c3de4047bf5415370077cd] 2024-11-18T20:13:44,199 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8eaa77b8f4c3de4047bf5415370077cd#B#compaction#348 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:13:44,199 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/.tmp/B/b056b619409e4fa080cfd13b3b919ad3 is 50, key is test_row_0/B:col10/1731960821721/Put/seqid=0 2024-11-18T20:13:44,200 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411187f128565ff8d4f5ca38f0981e6df026e_8eaa77b8f4c3de4047bf5415370077cd, store=[table=TestAcidGuarantees family=A region=8eaa77b8f4c3de4047bf5415370077cd] 2024-11-18T20:13:44,200 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411187f128565ff8d4f5ca38f0981e6df026e_8eaa77b8f4c3de4047bf5415370077cd because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=8eaa77b8f4c3de4047bf5415370077cd] 2024-11-18T20:13:44,211 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742234_1410 (size=12697) 2024-11-18T20:13:44,212 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742235_1411 (size=4469) 2024-11-18T20:13:44,216 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/.tmp/B/b056b619409e4fa080cfd13b3b919ad3 as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/B/b056b619409e4fa080cfd13b3b919ad3 2024-11-18T20:13:44,220 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8eaa77b8f4c3de4047bf5415370077cd/B of 8eaa77b8f4c3de4047bf5415370077cd into b056b619409e4fa080cfd13b3b919ad3(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:13:44,220 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8eaa77b8f4c3de4047bf5415370077cd: 2024-11-18T20:13:44,220 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., storeName=8eaa77b8f4c3de4047bf5415370077cd/B, priority=13, startTime=1731960824186; duration=0sec 2024-11-18T20:13:44,220 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T20:13:44,220 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8eaa77b8f4c3de4047bf5415370077cd:B 2024-11-18T20:13:44,220 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T20:13:44,221 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T20:13:44,221 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HStore(1541): 8eaa77b8f4c3de4047bf5415370077cd/C is initiating minor compaction (all files) 2024-11-18T20:13:44,221 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8eaa77b8f4c3de4047bf5415370077cd/C in TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd. 2024-11-18T20:13:44,221 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/C/941c6ba825684c128518e0753f084de3, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/C/c2150536f53a48afb12ba0bbcef65e30, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/C/3e4de34e127544cba9c7b19533d00c3f] into tmpdir=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/.tmp, totalSize=36.0 K 2024-11-18T20:13:44,221 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] compactions.Compactor(225): Compacting 941c6ba825684c128518e0753f084de3, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=183, earliestPutTs=1731960819620 2024-11-18T20:13:44,221 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] compactions.Compactor(225): Compacting c2150536f53a48afb12ba0bbcef65e30, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=209, earliestPutTs=1731960820343 2024-11-18T20:13:44,221 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3e4de34e127544cba9c7b19533d00c3f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=223, earliestPutTs=1731960821721 2024-11-18T20:13:44,226 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8eaa77b8f4c3de4047bf5415370077cd#C#compaction#349 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:13:44,226 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/.tmp/C/c235d8d94536475a9d237e3f0a65b6af is 50, key is test_row_0/C:col10/1731960821721/Put/seqid=0 2024-11-18T20:13:44,230 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742236_1412 (size=12697) 2024-11-18T20:13:44,233 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/.tmp/C/c235d8d94536475a9d237e3f0a65b6af as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/C/c235d8d94536475a9d237e3f0a65b6af 2024-11-18T20:13:44,237 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8eaa77b8f4c3de4047bf5415370077cd/C of 8eaa77b8f4c3de4047bf5415370077cd into c235d8d94536475a9d237e3f0a65b6af(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:13:44,237 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8eaa77b8f4c3de4047bf5415370077cd: 2024-11-18T20:13:44,237 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., storeName=8eaa77b8f4c3de4047bf5415370077cd/C, priority=13, startTime=1731960824187; duration=0sec 2024-11-18T20:13:44,237 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:13:44,237 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8eaa77b8f4c3de4047bf5415370077cd:C 2024-11-18T20:13:44,240 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35963 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-18T20:13:44,240 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd. 2024-11-18T20:13:44,240 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2902): Flushing 8eaa77b8f4c3de4047bf5415370077cd 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-18T20:13:44,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118bbf6203c7ab24b2ea960ea16a99f1e5b_8eaa77b8f4c3de4047bf5415370077cd is 50, key is test_row_0/A:col10/1731960822597/Put/seqid=0 2024-11-18T20:13:44,250 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742237_1413 (size=12304) 2024-11-18T20:13:44,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:44,254 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118bbf6203c7ab24b2ea960ea16a99f1e5b_8eaa77b8f4c3de4047bf5415370077cd to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118bbf6203c7ab24b2ea960ea16a99f1e5b_8eaa77b8f4c3de4047bf5415370077cd 2024-11-18T20:13:44,255 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/.tmp/A/cac157a8d0be4d8294b608033637571f, store: [table=TestAcidGuarantees family=A region=8eaa77b8f4c3de4047bf5415370077cd] 2024-11-18T20:13:44,255 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/.tmp/A/cac157a8d0be4d8294b608033637571f is 175, key is test_row_0/A:col10/1731960822597/Put/seqid=0 2024-11-18T20:13:44,259 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742238_1414 (size=31105) 2024-11-18T20:13:44,613 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8eaa77b8f4c3de4047bf5415370077cd#A#compaction#347 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:13:44,613 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/.tmp/A/cf8d6b0ce786423db2e2f5c6fd4abf78 is 175, key is test_row_0/A:col10/1731960821721/Put/seqid=0 2024-11-18T20:13:44,617 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742239_1415 (size=31651) 2024-11-18T20:13:44,622 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/.tmp/A/cf8d6b0ce786423db2e2f5c6fd4abf78 as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/A/cf8d6b0ce786423db2e2f5c6fd4abf78 2024-11-18T20:13:44,626 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8eaa77b8f4c3de4047bf5415370077cd/A of 8eaa77b8f4c3de4047bf5415370077cd into cf8d6b0ce786423db2e2f5c6fd4abf78(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:13:44,626 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8eaa77b8f4c3de4047bf5415370077cd: 2024-11-18T20:13:44,626 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., storeName=8eaa77b8f4c3de4047bf5415370077cd/A, priority=13, startTime=1731960824186; duration=0sec 2024-11-18T20:13:44,626 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:13:44,626 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8eaa77b8f4c3de4047bf5415370077cd:A 2024-11-18T20:13:44,660 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=249, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/.tmp/A/cac157a8d0be4d8294b608033637571f 2024-11-18T20:13:44,666 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/.tmp/B/ded377de394f42e5a167b0ca1009ddac is 50, key is test_row_0/B:col10/1731960822597/Put/seqid=0 2024-11-18T20:13:44,669 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742240_1416 (size=12151) 2024-11-18T20:13:44,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(8855): Flush requested on 8eaa77b8f4c3de4047bf5415370077cd 2024-11-18T20:13:44,757 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd. as already flushing 2024-11-18T20:13:44,792 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:44,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39882 deadline: 1731960884784, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:44,792 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:44,792 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:44,792 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:13:44,793 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:44,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39924 deadline: 1731960884791, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:44,793 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:44,793 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:44,793 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:13:44,906 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:44,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39882 deadline: 1731960884903, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:44,907 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:44,907 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:44,907 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:44,907 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:13:44,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39924 deadline: 1731960884903, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:44,907 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:44,907 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:44,907 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:13:45,070 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=249 (bloomFilter=true), to=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/.tmp/B/ded377de394f42e5a167b0ca1009ddac 2024-11-18T20:13:45,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/.tmp/C/728d736b72a941109de410de2532a12f is 50, key is test_row_0/C:col10/1731960822597/Put/seqid=0 2024-11-18T20:13:45,078 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742241_1417 (size=12151) 2024-11-18T20:13:45,079 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=249 (bloomFilter=true), to=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/.tmp/C/728d736b72a941109de410de2532a12f 2024-11-18T20:13:45,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/.tmp/A/cac157a8d0be4d8294b608033637571f as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/A/cac157a8d0be4d8294b608033637571f 2024-11-18T20:13:45,085 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/A/cac157a8d0be4d8294b608033637571f, entries=150, sequenceid=249, filesize=30.4 K 2024-11-18T20:13:45,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/.tmp/B/ded377de394f42e5a167b0ca1009ddac as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/B/ded377de394f42e5a167b0ca1009ddac 2024-11-18T20:13:45,089 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/B/ded377de394f42e5a167b0ca1009ddac, entries=150, sequenceid=249, filesize=11.9 K 2024-11-18T20:13:45,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/.tmp/C/728d736b72a941109de410de2532a12f as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/C/728d736b72a941109de410de2532a12f 2024-11-18T20:13:45,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:45,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:45,092 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/C/728d736b72a941109de410de2532a12f, entries=150, sequenceid=249, filesize=11.9 K 2024-11-18T20:13:45,093 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for 8eaa77b8f4c3de4047bf5415370077cd in 853ms, sequenceid=249, compaction requested=false 2024-11-18T20:13:45,093 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2603): Flush status journal for 8eaa77b8f4c3de4047bf5415370077cd: 2024-11-18T20:13:45,093 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd. 2024-11-18T20:13:45,093 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=108 2024-11-18T20:13:45,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.HMaster(4169): Remote procedure done, pid=108 2024-11-18T20:13:45,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:45,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:45,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:45,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:45,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:45,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:45,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:45,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:45,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:45,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:45,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:45,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:45,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:45,096 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=108, resume processing ppid=107 2024-11-18T20:13:45,096 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=108, ppid=107, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.4480 sec 2024-11-18T20:13:45,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:45,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:45,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:45,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:45,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:45,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:45,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:45,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:45,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:45,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:45,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:45,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:45,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:45,098 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=107, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees in 3.4530 sec 2024-11-18T20:13:45,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:45,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:45,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:45,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:45,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:45,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:45,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:45,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:45,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:45,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:45,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:45,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:45,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:45,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:45,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:45,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:45,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:45,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:45,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:45,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:45,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:45,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:45,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:45,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:45,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:45,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:45,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:45,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:45,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:45,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:45,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:45,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:45,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:45,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:45,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:45,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:45,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:45,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:45,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:45,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:45,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:45,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:45,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:45,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:45,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:45,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:45,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:45,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:45,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:45,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:45,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:45,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:45,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:45,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:45,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:45,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:45,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:45,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:45,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:45,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:45,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:45,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:45,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:45,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:45,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:45,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:45,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:45,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:45,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:45,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:45,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:45,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:45,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:45,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:45,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:45,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:45,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:45,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:45,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:45,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:45,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:45,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:45,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:45,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:45,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:45,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:45,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:45,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:45,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:45,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:45,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:45,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:45,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:45,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:45,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:45,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:45,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:45,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:45,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:45,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:45,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:45,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:45,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:45,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:45,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:45,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:45,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:45,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:45,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:45,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:45,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:45,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:45,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:45,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:45,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:45,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:45,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:45,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:45,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:45,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:45,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:45,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:45,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:45,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:45,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:45,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:45,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:45,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:45,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:45,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:45,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:45,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:45,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:45,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:45,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:45,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(8855): Flush requested on 8eaa77b8f4c3de4047bf5415370077cd 2024-11-18T20:13:45,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:45,122 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-18T20:13:45,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:45,123 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8eaa77b8f4c3de4047bf5415370077cd 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-18T20:13:45,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:45,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:45,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:45,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:45,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:45,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:45,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:45,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:45,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:45,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:45,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:45,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:45,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:45,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:45,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:45,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:45,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:45,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:45,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:45,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:45,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:45,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:45,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:45,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:45,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:45,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:45,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:45,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:45,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:45,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:45,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:45,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:45,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:45,131 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118945f9a1dad114c1abd2374b09f1040e5_8eaa77b8f4c3de4047bf5415370077cd is 50, key is test_row_1/A:col10/1731960825122/Put/seqid=0 2024-11-18T20:13:45,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:45,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:45,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:45,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:45,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:45,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:45,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:45,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:45,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:45,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:45,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:45,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:45,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:45,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:45,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:45,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:45,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:45,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:45,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:45,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:45,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:45,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:45,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:45,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:45,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:45,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:45,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:45,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:45,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:45,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:45,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:45,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:45,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:45,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:45,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:45,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:45,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:45,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:45,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:45,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:45,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:45,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:45,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:45,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:45,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:45,143 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742242_1418 (size=12404) 2024-11-18T20:13:45,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:45,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:45,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:45,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:45,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:45,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:45,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:45,197 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:45,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39924 deadline: 1731960885194, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:45,198 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:45,198 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:45,198 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:13:45,201 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:45,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39882 deadline: 1731960885197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:45,201 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:45,201 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:45,201 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:13:45,309 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:45,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39924 deadline: 1731960885303, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:45,309 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:45,309 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:45,309 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:13:45,316 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:45,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39882 deadline: 1731960885313, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:45,317 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:45,317 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:45,317 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:13:45,518 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:45,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39924 deadline: 1731960885513, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:45,519 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:45,519 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:45,519 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:13:45,528 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:45,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39882 deadline: 1731960885523, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:45,528 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:45,528 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:45,528 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:13:45,543 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:45,546 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118945f9a1dad114c1abd2374b09f1040e5_8eaa77b8f4c3de4047bf5415370077cd to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118945f9a1dad114c1abd2374b09f1040e5_8eaa77b8f4c3de4047bf5415370077cd 2024-11-18T20:13:45,547 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/.tmp/A/f9452fe87d2d47cebf1c15266ad1b782, store: [table=TestAcidGuarantees family=A region=8eaa77b8f4c3de4047bf5415370077cd] 2024-11-18T20:13:45,547 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/.tmp/A/f9452fe87d2d47cebf1c15266ad1b782 is 175, key is test_row_1/A:col10/1731960825122/Put/seqid=0 2024-11-18T20:13:45,550 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742243_1419 (size=31201) 2024-11-18T20:13:45,551 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=263, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/.tmp/A/f9452fe87d2d47cebf1c15266ad1b782 2024-11-18T20:13:45,556 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/.tmp/B/2ffa04d7665a4b00837ccf092d337a3a is 50, key is test_row_1/B:col10/1731960825122/Put/seqid=0 2024-11-18T20:13:45,559 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742244_1420 (size=9857) 2024-11-18T20:13:45,560 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=263 (bloomFilter=true), to=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/.tmp/B/2ffa04d7665a4b00837ccf092d337a3a 2024-11-18T20:13:45,566 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/.tmp/C/26e9513e9840456689d3993329e2b085 is 50, key is test_row_1/C:col10/1731960825122/Put/seqid=0 2024-11-18T20:13:45,569 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742245_1421 (size=9857) 2024-11-18T20:13:45,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-18T20:13:45,823 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-18T20:13:45,824 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-18T20:13:45,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] procedure2.ProcedureExecutor(1139): Stored pid=109, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees 2024-11-18T20:13:45,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-18T20:13:45,826 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=109, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-18T20:13:45,827 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=109, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-18T20:13:45,827 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=110, ppid=109, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-18T20:13:45,827 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:45,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39924 deadline: 1731960885823, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:45,828 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:45,828 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:45,828 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:13:45,839 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:45,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39882 deadline: 1731960885833, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:45,839 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:45,839 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:45,839 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:13:45,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-18T20:13:45,938 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:45,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39866 deadline: 1731960885933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:45,939 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to c0a89b2656d4,35963,1731960718223 for 'test_row_2' in 8eaa77b8f4c3de4047bf5415370077cd of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4200 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-18T20:13:45,939 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:45,939 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:45,939 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:13:45,970 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=263 (bloomFilter=true), to=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/.tmp/C/26e9513e9840456689d3993329e2b085 2024-11-18T20:13:45,973 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/.tmp/A/f9452fe87d2d47cebf1c15266ad1b782 as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/A/f9452fe87d2d47cebf1c15266ad1b782 2024-11-18T20:13:45,978 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/A/f9452fe87d2d47cebf1c15266ad1b782, entries=150, sequenceid=263, filesize=30.5 K 2024-11-18T20:13:45,979 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35963 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-18T20:13:45,979 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd. 2024-11-18T20:13:45,979 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd. as already flushing 2024-11-18T20:13:45,979 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd. 2024-11-18T20:13:45,979 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 8eaa77b8f4c3de4047bf5415370077cd, NAME => 'TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:13:45,979 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 8eaa77b8f4c3de4047bf5415370077cd, NAME => 'TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:13:45,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8eaa77b8f4c3de4047bf5415370077cd, NAME => 'TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8eaa77b8f4c3de4047bf5415370077cd, NAME => 'TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:13:45,981 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/.tmp/B/2ffa04d7665a4b00837ccf092d337a3a as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/B/2ffa04d7665a4b00837ccf092d337a3a 2024-11-18T20:13:45,984 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/B/2ffa04d7665a4b00837ccf092d337a3a, entries=100, sequenceid=263, filesize=9.6 K 2024-11-18T20:13:45,985 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/.tmp/C/26e9513e9840456689d3993329e2b085 as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/C/26e9513e9840456689d3993329e2b085 2024-11-18T20:13:45,988 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/C/26e9513e9840456689d3993329e2b085, entries=100, sequenceid=263, filesize=9.6 K 2024-11-18T20:13:45,988 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for 8eaa77b8f4c3de4047bf5415370077cd in 866ms, sequenceid=263, compaction requested=true 2024-11-18T20:13:45,988 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8eaa77b8f4c3de4047bf5415370077cd: 2024-11-18T20:13:45,989 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8eaa77b8f4c3de4047bf5415370077cd:A, priority=-2147483648, current under compaction store size is 1 2024-11-18T20:13:45,989 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T20:13:45,989 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:13:45,989 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8eaa77b8f4c3de4047bf5415370077cd:B, priority=-2147483648, current under compaction store size is 2 2024-11-18T20:13:45,989 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:13:45,989 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T20:13:45,989 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8eaa77b8f4c3de4047bf5415370077cd:C, priority=-2147483648, current under compaction store size is 3 2024-11-18T20:13:45,989 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T20:13:45,989 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 93957 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T20:13:45,989 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34705 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T20:13:45,989 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HStore(1541): 8eaa77b8f4c3de4047bf5415370077cd/B is initiating minor compaction (all files) 2024-11-18T20:13:45,989 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HStore(1541): 8eaa77b8f4c3de4047bf5415370077cd/A is initiating minor compaction (all files) 2024-11-18T20:13:45,989 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8eaa77b8f4c3de4047bf5415370077cd/A in TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd. 2024-11-18T20:13:45,989 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8eaa77b8f4c3de4047bf5415370077cd/B in TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd. 2024-11-18T20:13:45,990 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/B/b056b619409e4fa080cfd13b3b919ad3, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/B/ded377de394f42e5a167b0ca1009ddac, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/B/2ffa04d7665a4b00837ccf092d337a3a] into tmpdir=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/.tmp, totalSize=33.9 K 2024-11-18T20:13:45,990 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/A/cf8d6b0ce786423db2e2f5c6fd4abf78, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/A/cac157a8d0be4d8294b608033637571f, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/A/f9452fe87d2d47cebf1c15266ad1b782] into tmpdir=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/.tmp, totalSize=91.8 K 2024-11-18T20:13:45,990 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd. 2024-11-18T20:13:45,990 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd. files: [hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/A/cf8d6b0ce786423db2e2f5c6fd4abf78, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/A/cac157a8d0be4d8294b608033637571f, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/A/f9452fe87d2d47cebf1c15266ad1b782] 2024-11-18T20:13:45,990 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.Compactor(225): Compacting b056b619409e4fa080cfd13b3b919ad3, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=223, earliestPutTs=1731960821721 2024-11-18T20:13:45,990 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] compactions.Compactor(225): Compacting cf8d6b0ce786423db2e2f5c6fd4abf78, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=223, earliestPutTs=1731960821721 2024-11-18T20:13:45,990 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.Compactor(225): Compacting ded377de394f42e5a167b0ca1009ddac, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=249, earliestPutTs=1731960822595 2024-11-18T20:13:45,990 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] compactions.Compactor(225): Compacting cac157a8d0be4d8294b608033637571f, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=249, earliestPutTs=1731960822595 2024-11-18T20:13:45,990 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.Compactor(225): Compacting 2ffa04d7665a4b00837ccf092d337a3a, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=263, earliestPutTs=1731960824789 2024-11-18T20:13:45,991 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] compactions.Compactor(225): Compacting f9452fe87d2d47cebf1c15266ad1b782, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=263, earliestPutTs=1731960824763 2024-11-18T20:13:45,995 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=8eaa77b8f4c3de4047bf5415370077cd] 2024-11-18T20:13:45,996 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8eaa77b8f4c3de4047bf5415370077cd#B#compaction#356 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:13:45,996 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/.tmp/B/967221e31fa3415f8d378c7fcdf65b68 is 50, key is test_row_0/B:col10/1731960822597/Put/seqid=0 2024-11-18T20:13:45,996 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024111800e01a4d44cc4b2fa07a72173dc47794_8eaa77b8f4c3de4047bf5415370077cd store=[table=TestAcidGuarantees family=A region=8eaa77b8f4c3de4047bf5415370077cd] 2024-11-18T20:13:45,997 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024111800e01a4d44cc4b2fa07a72173dc47794_8eaa77b8f4c3de4047bf5415370077cd, store=[table=TestAcidGuarantees family=A region=8eaa77b8f4c3de4047bf5415370077cd] 2024-11-18T20:13:45,997 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111800e01a4d44cc4b2fa07a72173dc47794_8eaa77b8f4c3de4047bf5415370077cd because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=8eaa77b8f4c3de4047bf5415370077cd] 2024-11-18T20:13:46,001 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742246_1422 (size=12899) 2024-11-18T20:13:46,009 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742247_1423 (size=4469) 2024-11-18T20:13:46,131 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35963 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-18T20:13:46,132 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd. 2024-11-18T20:13:46,132 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2902): Flushing 8eaa77b8f4c3de4047bf5415370077cd 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-18T20:13:46,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118eadf2ac8ee30428f8e305613dbf7db51_8eaa77b8f4c3de4047bf5415370077cd is 50, key is test_row_0/A:col10/1731960825195/Put/seqid=0 2024-11-18T20:13:46,140 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742248_1424 (size=12454) 2024-11-18T20:13:46,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-18T20:13:46,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(8855): Flush requested on 8eaa77b8f4c3de4047bf5415370077cd 2024-11-18T20:13:46,349 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd. as already flushing 2024-11-18T20:13:46,380 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:46,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39924 deadline: 1731960886371, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:46,381 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:46,381 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:46,381 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:13:46,381 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:46,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39882 deadline: 1731960886376, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:46,382 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:46,382 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:46,382 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:13:46,403 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8eaa77b8f4c3de4047bf5415370077cd#A#compaction#357 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:13:46,404 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/.tmp/A/f8119bc32b434411a0805be444b0a7fb is 175, key is test_row_0/A:col10/1731960822597/Put/seqid=0 2024-11-18T20:13:46,407 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/.tmp/B/967221e31fa3415f8d378c7fcdf65b68 as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/B/967221e31fa3415f8d378c7fcdf65b68 2024-11-18T20:13:46,408 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742249_1425 (size=31960) 2024-11-18T20:13:46,411 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8eaa77b8f4c3de4047bf5415370077cd/B of 8eaa77b8f4c3de4047bf5415370077cd into 967221e31fa3415f8d378c7fcdf65b68(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:13:46,411 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8eaa77b8f4c3de4047bf5415370077cd: 2024-11-18T20:13:46,411 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., storeName=8eaa77b8f4c3de4047bf5415370077cd/B, priority=13, startTime=1731960825989; duration=0sec 2024-11-18T20:13:46,412 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T20:13:46,412 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8eaa77b8f4c3de4047bf5415370077cd:B 2024-11-18T20:13:46,412 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T20:13:46,412 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34705 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T20:13:46,412 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HStore(1541): 8eaa77b8f4c3de4047bf5415370077cd/C is initiating minor compaction (all files) 2024-11-18T20:13:46,412 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8eaa77b8f4c3de4047bf5415370077cd/C in TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd. 2024-11-18T20:13:46,412 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/C/c235d8d94536475a9d237e3f0a65b6af, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/C/728d736b72a941109de410de2532a12f, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/C/26e9513e9840456689d3993329e2b085] into tmpdir=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/.tmp, totalSize=33.9 K 2024-11-18T20:13:46,413 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/.tmp/A/f8119bc32b434411a0805be444b0a7fb as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/A/f8119bc32b434411a0805be444b0a7fb 2024-11-18T20:13:46,413 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.Compactor(225): Compacting c235d8d94536475a9d237e3f0a65b6af, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=223, earliestPutTs=1731960821721 2024-11-18T20:13:46,413 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.Compactor(225): Compacting 728d736b72a941109de410de2532a12f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=249, earliestPutTs=1731960822595 2024-11-18T20:13:46,413 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.Compactor(225): Compacting 26e9513e9840456689d3993329e2b085, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=263, earliestPutTs=1731960824789 2024-11-18T20:13:46,417 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8eaa77b8f4c3de4047bf5415370077cd/A of 8eaa77b8f4c3de4047bf5415370077cd into f8119bc32b434411a0805be444b0a7fb(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:13:46,417 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8eaa77b8f4c3de4047bf5415370077cd: 2024-11-18T20:13:46,417 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., storeName=8eaa77b8f4c3de4047bf5415370077cd/A, priority=13, startTime=1731960825988; duration=0sec 2024-11-18T20:13:46,417 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:13:46,417 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8eaa77b8f4c3de4047bf5415370077cd:A 2024-11-18T20:13:46,419 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8eaa77b8f4c3de4047bf5415370077cd#C#compaction#359 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:13:46,419 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/.tmp/C/c52d6bba54444cd3a22c51c225b0c781 is 50, key is test_row_0/C:col10/1731960822597/Put/seqid=0 2024-11-18T20:13:46,422 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742250_1426 (size=12899) 2024-11-18T20:13:46,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-18T20:13:46,487 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:46,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39924 deadline: 1731960886483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:46,487 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:46,487 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:46,487 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:46,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39882 deadline: 1731960886483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:46,487 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:13:46,487 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:46,487 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:46,487 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:13:46,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:46,545 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118eadf2ac8ee30428f8e305613dbf7db51_8eaa77b8f4c3de4047bf5415370077cd to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118eadf2ac8ee30428f8e305613dbf7db51_8eaa77b8f4c3de4047bf5415370077cd 2024-11-18T20:13:46,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/.tmp/A/6484aa56d8e04365921fd49f19d2f396, store: [table=TestAcidGuarantees family=A region=8eaa77b8f4c3de4047bf5415370077cd] 2024-11-18T20:13:46,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/.tmp/A/6484aa56d8e04365921fd49f19d2f396 is 175, key is test_row_0/A:col10/1731960825195/Put/seqid=0 2024-11-18T20:13:46,562 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742251_1427 (size=31255) 2024-11-18T20:13:46,564 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=286, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/.tmp/A/6484aa56d8e04365921fd49f19d2f396 2024-11-18T20:13:46,573 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/.tmp/B/d418fff74b2d41e9a669c3e6aea22928 is 50, key is test_row_0/B:col10/1731960825195/Put/seqid=0 2024-11-18T20:13:46,583 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742252_1428 (size=12301) 2024-11-18T20:13:46,702 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:46,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39924 deadline: 1731960886693, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:46,702 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:46,702 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:46,702 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:46,702 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:13:46,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39882 deadline: 1731960886693, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:46,702 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:46,703 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:46,703 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:13:46,826 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/.tmp/C/c52d6bba54444cd3a22c51c225b0c781 as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/C/c52d6bba54444cd3a22c51c225b0c781 2024-11-18T20:13:46,829 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8eaa77b8f4c3de4047bf5415370077cd/C of 8eaa77b8f4c3de4047bf5415370077cd into c52d6bba54444cd3a22c51c225b0c781(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:13:46,829 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8eaa77b8f4c3de4047bf5415370077cd: 2024-11-18T20:13:46,829 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., storeName=8eaa77b8f4c3de4047bf5415370077cd/C, priority=13, startTime=1731960825989; duration=0sec 2024-11-18T20:13:46,829 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:13:46,829 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8eaa77b8f4c3de4047bf5415370077cd:C 2024-11-18T20:13:46,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-18T20:13:46,983 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=286 (bloomFilter=true), to=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/.tmp/B/d418fff74b2d41e9a669c3e6aea22928 2024-11-18T20:13:46,988 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/.tmp/C/9f3cdf7ab96c472a9ab3566c3a7dc863 is 50, key is test_row_0/C:col10/1731960825195/Put/seqid=0 2024-11-18T20:13:46,991 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742253_1429 (size=12301) 2024-11-18T20:13:47,014 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:47,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39924 deadline: 1731960887013, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:47,015 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:47,015 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:47,015 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:47,015 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:13:47,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39882 deadline: 1731960887013, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:47,015 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:47,015 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:47,015 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:13:47,392 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=286 (bloomFilter=true), to=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/.tmp/C/9f3cdf7ab96c472a9ab3566c3a7dc863 2024-11-18T20:13:47,396 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/.tmp/A/6484aa56d8e04365921fd49f19d2f396 as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/A/6484aa56d8e04365921fd49f19d2f396 2024-11-18T20:13:47,400 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/A/6484aa56d8e04365921fd49f19d2f396, entries=150, sequenceid=286, filesize=30.5 K 2024-11-18T20:13:47,401 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/.tmp/B/d418fff74b2d41e9a669c3e6aea22928 as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/B/d418fff74b2d41e9a669c3e6aea22928 2024-11-18T20:13:47,408 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/B/d418fff74b2d41e9a669c3e6aea22928, entries=150, sequenceid=286, filesize=12.0 K 2024-11-18T20:13:47,409 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/.tmp/C/9f3cdf7ab96c472a9ab3566c3a7dc863 as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/C/9f3cdf7ab96c472a9ab3566c3a7dc863 2024-11-18T20:13:47,414 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/C/9f3cdf7ab96c472a9ab3566c3a7dc863, entries=150, sequenceid=286, filesize=12.0 K 2024-11-18T20:13:47,415 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for 8eaa77b8f4c3de4047bf5415370077cd in 1283ms, sequenceid=286, compaction requested=false 2024-11-18T20:13:47,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2603): Flush status journal for 8eaa77b8f4c3de4047bf5415370077cd: 2024-11-18T20:13:47,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd. 2024-11-18T20:13:47,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=110 2024-11-18T20:13:47,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.HMaster(4169): Remote procedure done, pid=110 2024-11-18T20:13:47,419 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=110, resume processing ppid=109 2024-11-18T20:13:47,419 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=110, ppid=109, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5900 sec 2024-11-18T20:13:47,421 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=109, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees in 1.5960 sec 2024-11-18T20:13:47,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(8855): Flush requested on 8eaa77b8f4c3de4047bf5415370077cd 2024-11-18T20:13:47,530 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-18T20:13:47,530 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8eaa77b8f4c3de4047bf5415370077cd 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-18T20:13:47,537 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411189fa2a4ec64054691ae6ab627b13e4ab7_8eaa77b8f4c3de4047bf5415370077cd is 50, key is test_row_0/A:col10/1731960826370/Put/seqid=0 2024-11-18T20:13:47,542 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742254_1430 (size=14994) 2024-11-18T20:13:47,543 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:47,548 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411189fa2a4ec64054691ae6ab627b13e4ab7_8eaa77b8f4c3de4047bf5415370077cd to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411189fa2a4ec64054691ae6ab627b13e4ab7_8eaa77b8f4c3de4047bf5415370077cd 2024-11-18T20:13:47,549 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/.tmp/A/c72a8c34180b4c9d848eb6b85e6acca5, store: [table=TestAcidGuarantees family=A region=8eaa77b8f4c3de4047bf5415370077cd] 2024-11-18T20:13:47,549 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/.tmp/A/c72a8c34180b4c9d848eb6b85e6acca5 is 175, key is test_row_0/A:col10/1731960826370/Put/seqid=0 2024-11-18T20:13:47,553 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742255_1431 (size=39949) 2024-11-18T20:13:47,554 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=303, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/.tmp/A/c72a8c34180b4c9d848eb6b85e6acca5 2024-11-18T20:13:47,560 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/.tmp/B/9475a47b2d8943aea0759f94dc2ccdfb is 50, key is test_row_0/B:col10/1731960826370/Put/seqid=0 2024-11-18T20:13:47,570 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742256_1432 (size=12301) 2024-11-18T20:13:47,571 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=303 (bloomFilter=true), to=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/.tmp/B/9475a47b2d8943aea0759f94dc2ccdfb 2024-11-18T20:13:47,580 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/.tmp/C/1800cacd2c4b42b4ae32e3351cd8d1cb is 50, key is test_row_0/C:col10/1731960826370/Put/seqid=0 2024-11-18T20:13:47,582 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742257_1433 (size=12301) 2024-11-18T20:13:47,617 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:47,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39924 deadline: 1731960887608, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:47,617 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:47,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39882 deadline: 1731960887609, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:47,618 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:47,618 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:47,618 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:13:47,619 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:47,619 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:47,619 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:13:47,730 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:47,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39882 deadline: 1731960887723, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:47,730 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:47,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39924 deadline: 1731960887723, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:47,732 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:47,732 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:47,732 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:47,732 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:47,732 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:13:47,732 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:13:47,947 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:47,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39924 deadline: 1731960887943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:47,948 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:47,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39882 deadline: 1731960887943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:47,948 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:47,948 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:47,948 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:13:47,948 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:47,948 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:47,948 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:13:47,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-18T20:13:47,974 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-18T20:13:47,974 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-18T20:13:47,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] procedure2.ProcedureExecutor(1139): Stored pid=111, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees 2024-11-18T20:13:47,976 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=111, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-18T20:13:47,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-18T20:13:47,976 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=111, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-18T20:13:47,976 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=112, ppid=111, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-18T20:13:47,983 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=303 (bloomFilter=true), to=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/.tmp/C/1800cacd2c4b42b4ae32e3351cd8d1cb 2024-11-18T20:13:47,986 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/.tmp/A/c72a8c34180b4c9d848eb6b85e6acca5 as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/A/c72a8c34180b4c9d848eb6b85e6acca5 2024-11-18T20:13:47,989 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/A/c72a8c34180b4c9d848eb6b85e6acca5, entries=200, sequenceid=303, filesize=39.0 K 2024-11-18T20:13:47,990 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/.tmp/B/9475a47b2d8943aea0759f94dc2ccdfb as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/B/9475a47b2d8943aea0759f94dc2ccdfb 2024-11-18T20:13:47,993 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/B/9475a47b2d8943aea0759f94dc2ccdfb, entries=150, sequenceid=303, filesize=12.0 K 2024-11-18T20:13:47,993 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/.tmp/C/1800cacd2c4b42b4ae32e3351cd8d1cb as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/C/1800cacd2c4b42b4ae32e3351cd8d1cb 2024-11-18T20:13:47,996 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/C/1800cacd2c4b42b4ae32e3351cd8d1cb, entries=150, sequenceid=303, filesize=12.0 K 2024-11-18T20:13:47,997 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for 8eaa77b8f4c3de4047bf5415370077cd in 467ms, sequenceid=303, compaction requested=true 2024-11-18T20:13:47,997 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8eaa77b8f4c3de4047bf5415370077cd: 2024-11-18T20:13:47,997 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8eaa77b8f4c3de4047bf5415370077cd:A, priority=-2147483648, current under compaction store size is 1 2024-11-18T20:13:47,997 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:13:47,997 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T20:13:47,997 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8eaa77b8f4c3de4047bf5415370077cd:B, priority=-2147483648, current under compaction store size is 2 2024-11-18T20:13:47,997 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T20:13:47,997 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8eaa77b8f4c3de4047bf5415370077cd:C, priority=-2147483648, current under compaction store size is 3 2024-11-18T20:13:47,997 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-18T20:13:47,998 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T20:13:47,998 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 103164 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T20:13:47,998 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HStore(1541): 8eaa77b8f4c3de4047bf5415370077cd/A is initiating minor compaction (all files) 2024-11-18T20:13:47,998 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8eaa77b8f4c3de4047bf5415370077cd/A in TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd. 2024-11-18T20:13:47,998 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/A/f8119bc32b434411a0805be444b0a7fb, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/A/6484aa56d8e04365921fd49f19d2f396, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/A/c72a8c34180b4c9d848eb6b85e6acca5] into tmpdir=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/.tmp, totalSize=100.7 K 2024-11-18T20:13:47,998 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37501 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T20:13:47,998 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HStore(1541): 8eaa77b8f4c3de4047bf5415370077cd/B is initiating minor compaction (all files) 2024-11-18T20:13:47,998 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd. 2024-11-18T20:13:47,998 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd. files: [hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/A/f8119bc32b434411a0805be444b0a7fb, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/A/6484aa56d8e04365921fd49f19d2f396, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/A/c72a8c34180b4c9d848eb6b85e6acca5] 2024-11-18T20:13:47,998 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8eaa77b8f4c3de4047bf5415370077cd/B in TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd. 2024-11-18T20:13:47,999 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/B/967221e31fa3415f8d378c7fcdf65b68, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/B/d418fff74b2d41e9a669c3e6aea22928, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/B/9475a47b2d8943aea0759f94dc2ccdfb] into tmpdir=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/.tmp, totalSize=36.6 K 2024-11-18T20:13:47,999 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] compactions.Compactor(225): Compacting f8119bc32b434411a0805be444b0a7fb, keycount=150, bloomtype=ROW, size=31.2 K, encoding=NONE, compression=NONE, seqNum=263, earliestPutTs=1731960822597 2024-11-18T20:13:47,999 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.Compactor(225): Compacting 967221e31fa3415f8d378c7fcdf65b68, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=263, earliestPutTs=1731960822597 2024-11-18T20:13:47,999 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6484aa56d8e04365921fd49f19d2f396, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=286, earliestPutTs=1731960825184 2024-11-18T20:13:47,999 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.Compactor(225): Compacting d418fff74b2d41e9a669c3e6aea22928, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=286, earliestPutTs=1731960825184 2024-11-18T20:13:47,999 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.Compactor(225): Compacting 9475a47b2d8943aea0759f94dc2ccdfb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=303, earliestPutTs=1731960826365 2024-11-18T20:13:47,999 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] compactions.Compactor(225): Compacting c72a8c34180b4c9d848eb6b85e6acca5, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=303, earliestPutTs=1731960826365 2024-11-18T20:13:48,007 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8eaa77b8f4c3de4047bf5415370077cd#B#compaction#365 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:13:48,008 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/.tmp/B/22d80fbc07a94cdba06fe23fcefd457d is 50, key is test_row_0/B:col10/1731960826370/Put/seqid=0 2024-11-18T20:13:48,009 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=8eaa77b8f4c3de4047bf5415370077cd] 2024-11-18T20:13:48,011 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411182241630561e34c79925b9ccd90b39f2d_8eaa77b8f4c3de4047bf5415370077cd store=[table=TestAcidGuarantees family=A region=8eaa77b8f4c3de4047bf5415370077cd] 2024-11-18T20:13:48,013 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411182241630561e34c79925b9ccd90b39f2d_8eaa77b8f4c3de4047bf5415370077cd, store=[table=TestAcidGuarantees family=A region=8eaa77b8f4c3de4047bf5415370077cd] 2024-11-18T20:13:48,013 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411182241630561e34c79925b9ccd90b39f2d_8eaa77b8f4c3de4047bf5415370077cd because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=8eaa77b8f4c3de4047bf5415370077cd] 2024-11-18T20:13:48,014 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742258_1434 (size=13051) 2024-11-18T20:13:48,018 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742259_1435 (size=4469) 2024-11-18T20:13:48,019 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8eaa77b8f4c3de4047bf5415370077cd#A#compaction#366 average throughput is 2.44 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:13:48,019 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/.tmp/A/24337a4beed3414c9d6c55e64ad0b12a is 175, key is test_row_0/A:col10/1731960826370/Put/seqid=0 2024-11-18T20:13:48,026 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742260_1436 (size=32005) 2024-11-18T20:13:48,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-18T20:13:48,128 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35963 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-18T20:13:48,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd. 2024-11-18T20:13:48,129 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2902): Flushing 8eaa77b8f4c3de4047bf5415370077cd 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-18T20:13:48,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118a0e3cc7a8e6c404f954ba12bc50f5dbd_8eaa77b8f4c3de4047bf5415370077cd is 50, key is test_row_0/A:col10/1731960827607/Put/seqid=0 2024-11-18T20:13:48,143 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742261_1437 (size=12454) 2024-11-18T20:13:48,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(8855): Flush requested on 8eaa77b8f4c3de4047bf5415370077cd 2024-11-18T20:13:48,256 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd. as already flushing 2024-11-18T20:13:48,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-18T20:13:48,294 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:48,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39882 deadline: 1731960888285, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:48,295 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:48,295 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:48,295 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:13:48,304 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:48,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39924 deadline: 1731960888292, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:48,305 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:48,305 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:48,305 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:13:48,411 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:48,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39834 deadline: 1731960888403, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:48,411 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:48,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39882 deadline: 1731960888403, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:48,411 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to c0a89b2656d4,35963,1731960718223 for 'test_row_2' in 8eaa77b8f4c3de4047bf5415370077cd of TestAcidGuarantees failed, tries = 9, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 18314 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-18T20:13:48,412 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:48,412 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:48,412 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:13:48,412 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:48,412 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:48,413 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:13:48,417 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/.tmp/B/22d80fbc07a94cdba06fe23fcefd457d as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/B/22d80fbc07a94cdba06fe23fcefd457d 2024-11-18T20:13:48,421 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8eaa77b8f4c3de4047bf5415370077cd/B of 8eaa77b8f4c3de4047bf5415370077cd into 22d80fbc07a94cdba06fe23fcefd457d(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:13:48,421 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8eaa77b8f4c3de4047bf5415370077cd: 2024-11-18T20:13:48,421 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., storeName=8eaa77b8f4c3de4047bf5415370077cd/B, priority=13, startTime=1731960827997; duration=0sec 2024-11-18T20:13:48,421 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T20:13:48,421 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8eaa77b8f4c3de4047bf5415370077cd:B 2024-11-18T20:13:48,421 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T20:13:48,423 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37501 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T20:13:48,424 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HStore(1541): 8eaa77b8f4c3de4047bf5415370077cd/C is initiating minor compaction (all files) 2024-11-18T20:13:48,424 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8eaa77b8f4c3de4047bf5415370077cd/C in TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd. 2024-11-18T20:13:48,424 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:48,424 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/C/c52d6bba54444cd3a22c51c225b0c781, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/C/9f3cdf7ab96c472a9ab3566c3a7dc863, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/C/1800cacd2c4b42b4ae32e3351cd8d1cb] into tmpdir=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/.tmp, totalSize=36.6 K 2024-11-18T20:13:48,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39924 deadline: 1731960888413, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:48,424 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:48,424 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:48,424 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:13:48,430 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:48,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39910 deadline: 1731960888423, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:48,430 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to c0a89b2656d4,35963,1731960718223 for 'test_row_0' in 8eaa77b8f4c3de4047bf5415370077cd of TestAcidGuarantees failed, tries = 9, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 18338 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-18T20:13:48,431 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:48,431 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:48,431 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:13:48,433 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.Compactor(225): Compacting c52d6bba54444cd3a22c51c225b0c781, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=263, earliestPutTs=1731960822597 2024-11-18T20:13:48,434 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.Compactor(225): Compacting 9f3cdf7ab96c472a9ab3566c3a7dc863, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=286, earliestPutTs=1731960825184 2024-11-18T20:13:48,434 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.Compactor(225): Compacting 1800cacd2c4b42b4ae32e3351cd8d1cb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=303, earliestPutTs=1731960826365 2024-11-18T20:13:48,437 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/.tmp/A/24337a4beed3414c9d6c55e64ad0b12a as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/A/24337a4beed3414c9d6c55e64ad0b12a 2024-11-18T20:13:48,442 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8eaa77b8f4c3de4047bf5415370077cd/A of 8eaa77b8f4c3de4047bf5415370077cd into 24337a4beed3414c9d6c55e64ad0b12a(size=31.3 K), total size for store is 31.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:13:48,442 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8eaa77b8f4c3de4047bf5415370077cd: 2024-11-18T20:13:48,442 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., storeName=8eaa77b8f4c3de4047bf5415370077cd/A, priority=13, startTime=1731960827997; duration=0sec 2024-11-18T20:13:48,442 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:13:48,442 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8eaa77b8f4c3de4047bf5415370077cd:A 2024-11-18T20:13:48,455 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8eaa77b8f4c3de4047bf5415370077cd#C#compaction#368 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:13:48,455 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/.tmp/C/470c305ee02d4575ae185191d19be0f4 is 50, key is test_row_0/C:col10/1731960826370/Put/seqid=0 2024-11-18T20:13:48,467 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742262_1438 (size=13051) 2024-11-18T20:13:48,544 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:48,547 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118a0e3cc7a8e6c404f954ba12bc50f5dbd_8eaa77b8f4c3de4047bf5415370077cd to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118a0e3cc7a8e6c404f954ba12bc50f5dbd_8eaa77b8f4c3de4047bf5415370077cd 2024-11-18T20:13:48,548 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/.tmp/A/25c3e3b7591a4e6da6c39ea3436974c9, store: [table=TestAcidGuarantees family=A region=8eaa77b8f4c3de4047bf5415370077cd] 2024-11-18T20:13:48,548 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/.tmp/A/25c3e3b7591a4e6da6c39ea3436974c9 is 175, key is test_row_0/A:col10/1731960827607/Put/seqid=0 2024-11-18T20:13:48,551 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742263_1439 (size=31255) 2024-11-18T20:13:48,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-18T20:13:48,629 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:48,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39882 deadline: 1731960888623, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:48,630 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:48,630 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:48,630 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:13:48,638 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:48,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39924 deadline: 1731960888633, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:48,638 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:48,638 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:48,638 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:13:48,872 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/.tmp/C/470c305ee02d4575ae185191d19be0f4 as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/C/470c305ee02d4575ae185191d19be0f4 2024-11-18T20:13:48,877 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8eaa77b8f4c3de4047bf5415370077cd/C of 8eaa77b8f4c3de4047bf5415370077cd into 470c305ee02d4575ae185191d19be0f4(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:13:48,878 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8eaa77b8f4c3de4047bf5415370077cd: 2024-11-18T20:13:48,878 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., storeName=8eaa77b8f4c3de4047bf5415370077cd/C, priority=13, startTime=1731960827997; duration=0sec 2024-11-18T20:13:48,878 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:13:48,878 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8eaa77b8f4c3de4047bf5415370077cd:C 2024-11-18T20:13:48,935 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:48,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39882 deadline: 1731960888933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:48,936 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:48,936 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:48,936 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:13:48,948 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:48,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39924 deadline: 1731960888943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:48,948 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:48,948 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:48,948 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:13:48,952 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=325, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/.tmp/A/25c3e3b7591a4e6da6c39ea3436974c9 2024-11-18T20:13:48,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/.tmp/B/37dfa2010f574c718cf52aba6b8ff150 is 50, key is test_row_0/B:col10/1731960827607/Put/seqid=0 2024-11-18T20:13:48,963 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742264_1440 (size=12301) 2024-11-18T20:13:49,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-18T20:13:49,363 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=325 (bloomFilter=true), to=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/.tmp/B/37dfa2010f574c718cf52aba6b8ff150 2024-11-18T20:13:49,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/.tmp/C/739dfd966d6844699a10381a0fca4a7d is 50, key is test_row_0/C:col10/1731960827607/Put/seqid=0 2024-11-18T20:13:49,375 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742265_1441 (size=12301) 2024-11-18T20:13:49,447 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:49,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39882 deadline: 1731960889443, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:49,447 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:49,447 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:49,447 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:13:49,466 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:13:49,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39924 deadline: 1731960889463, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:49,467 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:49,467 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:13:49,467 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:13:49,775 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=325 (bloomFilter=true), to=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/.tmp/C/739dfd966d6844699a10381a0fca4a7d 2024-11-18T20:13:49,779 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/.tmp/A/25c3e3b7591a4e6da6c39ea3436974c9 as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/A/25c3e3b7591a4e6da6c39ea3436974c9 2024-11-18T20:13:49,781 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/A/25c3e3b7591a4e6da6c39ea3436974c9, entries=150, sequenceid=325, filesize=30.5 K 2024-11-18T20:13:49,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/.tmp/B/37dfa2010f574c718cf52aba6b8ff150 as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/B/37dfa2010f574c718cf52aba6b8ff150 2024-11-18T20:13:49,785 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/B/37dfa2010f574c718cf52aba6b8ff150, entries=150, sequenceid=325, filesize=12.0 K 2024-11-18T20:13:49,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/.tmp/C/739dfd966d6844699a10381a0fca4a7d as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/C/739dfd966d6844699a10381a0fca4a7d 2024-11-18T20:13:49,790 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/C/739dfd966d6844699a10381a0fca4a7d, entries=150, sequenceid=325, filesize=12.0 K 2024-11-18T20:13:49,790 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=73.80 KB/75570 for 8eaa77b8f4c3de4047bf5415370077cd in 1661ms, sequenceid=325, compaction requested=false 2024-11-18T20:13:49,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2603): Flush status journal for 8eaa77b8f4c3de4047bf5415370077cd: 2024-11-18T20:13:49,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd. 2024-11-18T20:13:49,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=112 2024-11-18T20:13:49,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.HMaster(4169): Remote procedure done, pid=112 2024-11-18T20:13:49,798 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=112, resume processing ppid=111 2024-11-18T20:13:49,798 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=112, ppid=111, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8160 sec 2024-11-18T20:13:49,800 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=111, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees in 1.8250 sec 2024-11-18T20:13:49,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(8855): Flush requested on 8eaa77b8f4c3de4047bf5415370077cd 2024-11-18T20:13:49,979 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-18T20:13:49,979 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8eaa77b8f4c3de4047bf5415370077cd 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-11-18T20:13:49,985 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411188581e71857c14365b7d6c279a6714438_8eaa77b8f4c3de4047bf5415370077cd is 50, key is test_row_0/A:col10/1731960828274/Put/seqid=0 2024-11-18T20:13:49,990 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742266_1442 (size=14994) 2024-11-18T20:13:50,031 INFO [Thread-1584 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1584. 2024-11-18T20:13:50,031 DEBUG [Thread-1584 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-18T20:13:50,031 DEBUG [Thread-1584 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:13:50,031 INFO [Thread-1569 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1569. 2024-11-18T20:13:50,031 DEBUG [Thread-1584 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:13:50,031 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T20:13:50,031 DEBUG [Thread-1569 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-18T20:13:50,031 DEBUG [Thread-1569 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:13:50,031 INFO [Thread-1582 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1582. 2024-11-18T20:13:50,031 DEBUG [Thread-1582 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-18T20:13:50,031 DEBUG [Thread-1569 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:13:50,031 DEBUG [Thread-1582 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:13:50,031 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T20:13:50,031 DEBUG [Thread-1582 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:13:50,031 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T20:13:50,034 INFO [Thread-1586 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1586. 2024-11-18T20:13:50,034 DEBUG [Thread-1586 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-18T20:13:50,034 DEBUG [Thread-1586 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:13:50,034 DEBUG [Thread-1586 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:13:50,034 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T20:13:50,039 INFO [Thread-1578 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1578. 2024-11-18T20:13:50,039 DEBUG [Thread-1578 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-18T20:13:50,039 DEBUG [Thread-1578 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:13:50,039 DEBUG [Thread-1578 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:13:50,039 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T20:13:50,041 INFO [Thread-1580 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1580. 2024-11-18T20:13:50,041 DEBUG [Thread-1580 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-18T20:13:50,041 DEBUG [Thread-1580 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:13:50,041 DEBUG [Thread-1580 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:13:50,041 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T20:13:50,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-18T20:13:50,123 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-18T20:13:50,392 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:50,400 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411188581e71857c14365b7d6c279a6714438_8eaa77b8f4c3de4047bf5415370077cd to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411188581e71857c14365b7d6c279a6714438_8eaa77b8f4c3de4047bf5415370077cd 2024-11-18T20:13:50,401 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/.tmp/A/77bb042f669d44bdbe1357482358c8ad, store: [table=TestAcidGuarantees family=A region=8eaa77b8f4c3de4047bf5415370077cd] 2024-11-18T20:13:50,402 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/.tmp/A/77bb042f669d44bdbe1357482358c8ad is 175, key is test_row_0/A:col10/1731960828274/Put/seqid=0 2024-11-18T20:13:50,406 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742267_1443 (size=39949) 2024-11-18T20:13:50,466 INFO [Thread-1571 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1571. 2024-11-18T20:13:50,466 DEBUG [Thread-1571 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-18T20:13:50,466 DEBUG [Thread-1571 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:13:50,466 DEBUG [Thread-1571 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:13:50,466 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T20:13:50,474 INFO [Thread-1573 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1573. 2024-11-18T20:13:50,474 DEBUG [Thread-1573 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-18T20:13:50,474 DEBUG [Thread-1573 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:13:50,474 DEBUG [Thread-1573 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:13:50,474 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T20:13:50,807 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=343, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/.tmp/A/77bb042f669d44bdbe1357482358c8ad 2024-11-18T20:13:50,817 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/.tmp/B/2ae04bbc55d54d728bc6e782196448a8 is 50, key is test_row_0/B:col10/1731960828274/Put/seqid=0 2024-11-18T20:13:50,821 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742268_1444 (size=12301) 2024-11-18T20:13:51,222 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=343 (bloomFilter=true), to=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/.tmp/B/2ae04bbc55d54d728bc6e782196448a8 2024-11-18T20:13:51,233 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/.tmp/C/c34255e9dac2474e9fe6d79ce34b8221 is 50, key is test_row_0/C:col10/1731960828274/Put/seqid=0 2024-11-18T20:13:51,237 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742269_1445 (size=12301) 2024-11-18T20:13:51,637 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=343 (bloomFilter=true), to=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/.tmp/C/c34255e9dac2474e9fe6d79ce34b8221 2024-11-18T20:13:51,642 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/.tmp/A/77bb042f669d44bdbe1357482358c8ad as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/A/77bb042f669d44bdbe1357482358c8ad 2024-11-18T20:13:51,645 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/A/77bb042f669d44bdbe1357482358c8ad, entries=200, sequenceid=343, filesize=39.0 K 2024-11-18T20:13:51,646 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/.tmp/B/2ae04bbc55d54d728bc6e782196448a8 as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/B/2ae04bbc55d54d728bc6e782196448a8 2024-11-18T20:13:51,649 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/B/2ae04bbc55d54d728bc6e782196448a8, entries=150, sequenceid=343, filesize=12.0 K 2024-11-18T20:13:51,649 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/.tmp/C/c34255e9dac2474e9fe6d79ce34b8221 as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/C/c34255e9dac2474e9fe6d79ce34b8221 2024-11-18T20:13:51,652 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/C/c34255e9dac2474e9fe6d79ce34b8221, entries=150, sequenceid=343, filesize=12.0 K 2024-11-18T20:13:51,653 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=67.09 KB/68700 for 8eaa77b8f4c3de4047bf5415370077cd in 1674ms, sequenceid=343, compaction requested=true 2024-11-18T20:13:51,653 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8eaa77b8f4c3de4047bf5415370077cd: 2024-11-18T20:13:51,653 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8eaa77b8f4c3de4047bf5415370077cd:A, priority=-2147483648, current under compaction store size is 1 2024-11-18T20:13:51,653 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:13:51,653 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8eaa77b8f4c3de4047bf5415370077cd:B, priority=-2147483648, current under compaction store size is 2 2024-11-18T20:13:51,653 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T20:13:51,653 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:13:51,653 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T20:13:51,653 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8eaa77b8f4c3de4047bf5415370077cd:C, priority=-2147483648, current under compaction store size is 3 2024-11-18T20:13:51,653 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T20:13:51,654 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 103209 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T20:13:51,654 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T20:13:51,654 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HStore(1541): 8eaa77b8f4c3de4047bf5415370077cd/A is initiating minor compaction (all files) 2024-11-18T20:13:51,654 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HStore(1541): 8eaa77b8f4c3de4047bf5415370077cd/B is initiating minor compaction (all files) 2024-11-18T20:13:51,654 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8eaa77b8f4c3de4047bf5415370077cd/A in TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd. 2024-11-18T20:13:51,654 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8eaa77b8f4c3de4047bf5415370077cd/B in TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd. 2024-11-18T20:13:51,654 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/B/22d80fbc07a94cdba06fe23fcefd457d, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/B/37dfa2010f574c718cf52aba6b8ff150, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/B/2ae04bbc55d54d728bc6e782196448a8] into tmpdir=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/.tmp, totalSize=36.8 K 2024-11-18T20:13:51,654 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/A/24337a4beed3414c9d6c55e64ad0b12a, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/A/25c3e3b7591a4e6da6c39ea3436974c9, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/A/77bb042f669d44bdbe1357482358c8ad] into tmpdir=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/.tmp, totalSize=100.8 K 2024-11-18T20:13:51,654 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd. 2024-11-18T20:13:51,654 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd. files: [hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/A/24337a4beed3414c9d6c55e64ad0b12a, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/A/25c3e3b7591a4e6da6c39ea3436974c9, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/A/77bb042f669d44bdbe1357482358c8ad] 2024-11-18T20:13:51,654 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.Compactor(225): Compacting 22d80fbc07a94cdba06fe23fcefd457d, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=303, earliestPutTs=1731960826365 2024-11-18T20:13:51,654 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] compactions.Compactor(225): Compacting 24337a4beed3414c9d6c55e64ad0b12a, keycount=150, bloomtype=ROW, size=31.3 K, encoding=NONE, compression=NONE, seqNum=303, earliestPutTs=1731960826365 2024-11-18T20:13:51,654 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.Compactor(225): Compacting 37dfa2010f574c718cf52aba6b8ff150, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=325, earliestPutTs=1731960827596 2024-11-18T20:13:51,655 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] compactions.Compactor(225): Compacting 25c3e3b7591a4e6da6c39ea3436974c9, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=325, earliestPutTs=1731960827596 2024-11-18T20:13:51,655 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.Compactor(225): Compacting 2ae04bbc55d54d728bc6e782196448a8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=343, earliestPutTs=1731960828274 2024-11-18T20:13:51,655 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] compactions.Compactor(225): Compacting 77bb042f669d44bdbe1357482358c8ad, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=343, earliestPutTs=1731960828274 2024-11-18T20:13:51,659 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=8eaa77b8f4c3de4047bf5415370077cd] 2024-11-18T20:13:51,660 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8eaa77b8f4c3de4047bf5415370077cd#B#compaction#374 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:13:51,660 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241118842ba0bc76244ca380077c58d79ea7f7_8eaa77b8f4c3de4047bf5415370077cd store=[table=TestAcidGuarantees family=A region=8eaa77b8f4c3de4047bf5415370077cd] 2024-11-18T20:13:51,660 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/.tmp/B/1feca162e4a14994bfa54671cdd8583c is 50, key is test_row_0/B:col10/1731960828274/Put/seqid=0 2024-11-18T20:13:51,667 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241118842ba0bc76244ca380077c58d79ea7f7_8eaa77b8f4c3de4047bf5415370077cd, store=[table=TestAcidGuarantees family=A region=8eaa77b8f4c3de4047bf5415370077cd] 2024-11-18T20:13:51,667 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118842ba0bc76244ca380077c58d79ea7f7_8eaa77b8f4c3de4047bf5415370077cd because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=8eaa77b8f4c3de4047bf5415370077cd] 2024-11-18T20:13:51,668 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742270_1446 (size=13153) 2024-11-18T20:13:51,674 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/.tmp/B/1feca162e4a14994bfa54671cdd8583c as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/B/1feca162e4a14994bfa54671cdd8583c 2024-11-18T20:13:51,677 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8eaa77b8f4c3de4047bf5415370077cd/B of 8eaa77b8f4c3de4047bf5415370077cd into 1feca162e4a14994bfa54671cdd8583c(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:13:51,677 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8eaa77b8f4c3de4047bf5415370077cd: 2024-11-18T20:13:51,677 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., storeName=8eaa77b8f4c3de4047bf5415370077cd/B, priority=13, startTime=1731960831653; duration=0sec 2024-11-18T20:13:51,677 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T20:13:51,677 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8eaa77b8f4c3de4047bf5415370077cd:B 2024-11-18T20:13:51,677 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T20:13:51,678 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T20:13:51,678 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HStore(1541): 8eaa77b8f4c3de4047bf5415370077cd/C is initiating minor compaction (all files) 2024-11-18T20:13:51,678 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8eaa77b8f4c3de4047bf5415370077cd/C in TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd. 2024-11-18T20:13:51,678 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/C/470c305ee02d4575ae185191d19be0f4, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/C/739dfd966d6844699a10381a0fca4a7d, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/C/c34255e9dac2474e9fe6d79ce34b8221] into tmpdir=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/.tmp, totalSize=36.8 K 2024-11-18T20:13:51,679 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.Compactor(225): Compacting 470c305ee02d4575ae185191d19be0f4, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=303, earliestPutTs=1731960826365 2024-11-18T20:13:51,679 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.Compactor(225): Compacting 739dfd966d6844699a10381a0fca4a7d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=325, earliestPutTs=1731960827596 2024-11-18T20:13:51,679 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.Compactor(225): Compacting c34255e9dac2474e9fe6d79ce34b8221, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=343, earliestPutTs=1731960828274 2024-11-18T20:13:51,682 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742271_1447 (size=4469) 2024-11-18T20:13:51,684 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8eaa77b8f4c3de4047bf5415370077cd#A#compaction#375 average throughput is 0.98 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:13:51,685 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/.tmp/A/c3241a38fc354bdf8ba9791b40d6366f is 175, key is test_row_0/A:col10/1731960828274/Put/seqid=0 2024-11-18T20:13:51,689 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8eaa77b8f4c3de4047bf5415370077cd#C#compaction#376 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:13:51,689 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/.tmp/C/3e6a00ad32094ecca8236623cdcf3d9b is 50, key is test_row_0/C:col10/1731960828274/Put/seqid=0 2024-11-18T20:13:51,694 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742272_1448 (size=32107) 2024-11-18T20:13:51,699 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742273_1449 (size=13153) 2024-11-18T20:13:52,099 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/.tmp/A/c3241a38fc354bdf8ba9791b40d6366f as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/A/c3241a38fc354bdf8ba9791b40d6366f 2024-11-18T20:13:52,104 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/.tmp/C/3e6a00ad32094ecca8236623cdcf3d9b as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/C/3e6a00ad32094ecca8236623cdcf3d9b 2024-11-18T20:13:52,104 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8eaa77b8f4c3de4047bf5415370077cd/A of 8eaa77b8f4c3de4047bf5415370077cd into c3241a38fc354bdf8ba9791b40d6366f(size=31.4 K), total size for store is 31.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:13:52,104 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8eaa77b8f4c3de4047bf5415370077cd: 2024-11-18T20:13:52,104 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., storeName=8eaa77b8f4c3de4047bf5415370077cd/A, priority=13, startTime=1731960831653; duration=0sec 2024-11-18T20:13:52,105 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:13:52,105 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8eaa77b8f4c3de4047bf5415370077cd:A 2024-11-18T20:13:52,108 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8eaa77b8f4c3de4047bf5415370077cd/C of 8eaa77b8f4c3de4047bf5415370077cd into 3e6a00ad32094ecca8236623cdcf3d9b(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:13:52,108 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8eaa77b8f4c3de4047bf5415370077cd: 2024-11-18T20:13:52,108 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., storeName=8eaa77b8f4c3de4047bf5415370077cd/C, priority=13, startTime=1731960831653; duration=0sec 2024-11-18T20:13:52,108 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:13:52,108 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8eaa77b8f4c3de4047bf5415370077cd:C 2024-11-18T20:13:56,534 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-18T20:13:58,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(8855): Flush requested on 8eaa77b8f4c3de4047bf5415370077cd 2024-11-18T20:13:58,497 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-18T20:13:58,497 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8eaa77b8f4c3de4047bf5415370077cd 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-18T20:13:58,498 INFO [Thread-1575 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1575. 2024-11-18T20:13:58,498 DEBUG [Thread-1575 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-18T20:13:58,498 DEBUG [Thread-1575 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:13:58,498 DEBUG [Thread-1575 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:13:58,498 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T20:13:58,505 INFO [Thread-1567 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1567. 2024-11-18T20:13:58,505 DEBUG [Thread-1567 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-18T20:13:58,505 DEBUG [Thread-1567 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:13:58,506 DEBUG [Thread-1567 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:13:58,506 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T20:13:58,506 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-18T20:13:58,506 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 9 2024-11-18T20:13:58,506 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 57 2024-11-18T20:13:58,506 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 103 2024-11-18T20:13:58,506 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 100 2024-11-18T20:13:58,506 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 7 2024-11-18T20:13:58,506 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-18T20:13:58,506 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-18T20:13:58,506 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1480 2024-11-18T20:13:58,506 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4438 rows 2024-11-18T20:13:58,506 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1493 2024-11-18T20:13:58,507 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4471 rows 2024-11-18T20:13:58,507 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1481 2024-11-18T20:13:58,507 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4436 rows 2024-11-18T20:13:58,507 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1484 2024-11-18T20:13:58,507 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4449 rows 2024-11-18T20:13:58,507 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1479 2024-11-18T20:13:58,507 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4434 rows 2024-11-18T20:13:58,507 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-18T20:13:58,507 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobScanAtomicity(AcidGuaranteesTestBase.java:127) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-18T20:13:58,507 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:13:58,507 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:13:58,507 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T20:13:58,508 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-18T20:13:58,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] procedure2.ProcedureExecutor(1139): Stored pid=113, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-18T20:13:58,510 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118e0229bb8b17b48c99770f1dda2191c5e_8eaa77b8f4c3de4047bf5415370077cd is 50, key is test_row_0/A:col10/1731960838494/Put/seqid=0 2024-11-18T20:13:58,512 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731960838512"}]},"ts":"1731960838512"} 2024-11-18T20:13:58,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-18T20:13:58,514 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-18T20:13:58,514 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-18T20:13:58,515 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=114, ppid=113, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-18T20:13:58,516 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=115, ppid=114, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=8eaa77b8f4c3de4047bf5415370077cd, UNASSIGN}] 2024-11-18T20:13:58,517 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742274_1450 (size=12454) 2024-11-18T20:13:58,517 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=115, ppid=114, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=8eaa77b8f4c3de4047bf5415370077cd, UNASSIGN 2024-11-18T20:13:58,518 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=115 updating hbase:meta row=8eaa77b8f4c3de4047bf5415370077cd, regionState=CLOSING, regionLocation=c0a89b2656d4,35963,1731960718223 2024-11-18T20:13:58,519 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=115, ppid=114, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=8eaa77b8f4c3de4047bf5415370077cd, UNASSIGN because future has completed 2024-11-18T20:13:58,520 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-18T20:13:58,520 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=116, ppid=115, state=RUNNABLE, hasLock=false; CloseRegionProcedure 8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223}] 2024-11-18T20:13:58,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-18T20:13:58,672 INFO [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] handler.UnassignRegionHandler(122): Close 8eaa77b8f4c3de4047bf5415370077cd 2024-11-18T20:13:58,672 DEBUG [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-18T20:13:58,672 DEBUG [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HRegion(1722): Closing 8eaa77b8f4c3de4047bf5415370077cd, disabling compactions & flushes 2024-11-18T20:13:58,672 DEBUG [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HRegion(1993): waiting for 0 compactions & cache flush to complete for region TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd. 2024-11-18T20:13:58,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-18T20:13:58,918 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:13:58,927 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118e0229bb8b17b48c99770f1dda2191c5e_8eaa77b8f4c3de4047bf5415370077cd to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118e0229bb8b17b48c99770f1dda2191c5e_8eaa77b8f4c3de4047bf5415370077cd 2024-11-18T20:13:58,929 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/.tmp/A/239f37fb9491422f989ff9785689a834, store: [table=TestAcidGuarantees family=A region=8eaa77b8f4c3de4047bf5415370077cd] 2024-11-18T20:13:58,929 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/.tmp/A/239f37fb9491422f989ff9785689a834 is 175, key is test_row_0/A:col10/1731960838494/Put/seqid=0 2024-11-18T20:13:58,934 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742275_1451 (size=31255) 2024-11-18T20:13:59,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-18T20:13:59,336 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=360, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/.tmp/A/239f37fb9491422f989ff9785689a834 2024-11-18T20:13:59,350 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/.tmp/B/044f39110875474aa77bd3a600c19d5b is 50, key is test_row_0/B:col10/1731960838494/Put/seqid=0 2024-11-18T20:13:59,354 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742276_1452 (size=12301) 2024-11-18T20:13:59,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-18T20:13:59,755 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=360 (bloomFilter=true), to=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/.tmp/B/044f39110875474aa77bd3a600c19d5b 2024-11-18T20:13:59,766 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/.tmp/C/0465edff8efa4236a100e313b8715862 is 50, key is test_row_0/C:col10/1731960838494/Put/seqid=0 2024-11-18T20:13:59,770 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742277_1453 (size=12301) 2024-11-18T20:14:00,171 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=360 (bloomFilter=true), to=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/.tmp/C/0465edff8efa4236a100e313b8715862 2024-11-18T20:14:00,181 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/.tmp/A/239f37fb9491422f989ff9785689a834 as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/A/239f37fb9491422f989ff9785689a834 2024-11-18T20:14:00,186 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/A/239f37fb9491422f989ff9785689a834, entries=150, sequenceid=360, filesize=30.5 K 2024-11-18T20:14:00,187 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/.tmp/B/044f39110875474aa77bd3a600c19d5b as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/B/044f39110875474aa77bd3a600c19d5b 2024-11-18T20:14:00,191 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/B/044f39110875474aa77bd3a600c19d5b, entries=150, sequenceid=360, filesize=12.0 K 2024-11-18T20:14:00,192 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/.tmp/C/0465edff8efa4236a100e313b8715862 as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/C/0465edff8efa4236a100e313b8715862 2024-11-18T20:14:00,196 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/C/0465edff8efa4236a100e313b8715862, entries=150, sequenceid=360, filesize=12.0 K 2024-11-18T20:14:00,197 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=6.71 KB/6870 for 8eaa77b8f4c3de4047bf5415370077cd in 1700ms, sequenceid=360, compaction requested=false 2024-11-18T20:14:00,197 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8eaa77b8f4c3de4047bf5415370077cd: 2024-11-18T20:14:00,198 INFO [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd. 2024-11-18T20:14:00,198 DEBUG [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd. 2024-11-18T20:14:00,198 DEBUG [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd. after waiting 0 ms 2024-11-18T20:14:00,198 DEBUG [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd. 2024-11-18T20:14:00,198 INFO [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HRegion(2902): Flushing 8eaa77b8f4c3de4047bf5415370077cd 3/3 column families, dataSize=6.71 KB heapSize=18.33 KB 2024-11-18T20:14:00,204 DEBUG [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118f8d1fa9da01c46999a59c298aeccd180_8eaa77b8f4c3de4047bf5415370077cd is 50, key is test_row_2/A:col10/1731960838503/Put/seqid=0 2024-11-18T20:14:00,209 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742278_1454 (size=7374) 2024-11-18T20:14:00,610 DEBUG [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:00,614 INFO [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118f8d1fa9da01c46999a59c298aeccd180_8eaa77b8f4c3de4047bf5415370077cd to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118f8d1fa9da01c46999a59c298aeccd180_8eaa77b8f4c3de4047bf5415370077cd 2024-11-18T20:14:00,614 DEBUG [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/.tmp/A/6a64ab9e2f9f435d90ed15fc18325943, store: [table=TestAcidGuarantees family=A region=8eaa77b8f4c3de4047bf5415370077cd] 2024-11-18T20:14:00,615 DEBUG [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/.tmp/A/6a64ab9e2f9f435d90ed15fc18325943 is 175, key is test_row_2/A:col10/1731960838503/Put/seqid=0 2024-11-18T20:14:00,618 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742279_1455 (size=13865) 2024-11-18T20:14:00,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-18T20:14:00,798 WARN [regionserver/c0a89b2656d4:0.Chore.1 {}] hbase.ExecutorStatusChore(69): RS_COMPACTED_FILES_DISCHARGER's size info, queued: 6, running: 1 2024-11-18T20:14:00,798 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c0a89b2656d4:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/A/13725551bd2245be8850dcc8a4559260, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/A/882382ffd1fc46d2ab2a16547d84b48b, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/A/6ea9a7c8ae2d4356a5bad1fa37d0ad60, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/A/bb7a6318ae714a60b837601adf470003, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/A/98343979eecd4758af30dcd70afe64f5, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/A/c2eee0845c9c4513bd7be1efb6304d9e, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/A/98e854168bb1409588a759becb8a64d9, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/A/95ce812f00ed4f3cbbdfb0528d61f7d1, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/A/586b29f729354327b15ca11a6051cc77, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/A/9359688205b24faf89fd3d55818f214f, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/A/b378673a59534d59b53e09a3fa6b7c48, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/A/a2fd1f59874841f984a78cecf53ebc9c, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/A/1cefe7769199475883e4e4971a63f26c, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/A/d300b651488647d6bf3c7662bb435e6d, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/A/6eb81deeae224aa496c3e80fd807719c, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/A/e53f36ebfc4c42ecac30d3885f453e36, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/A/cf8d6b0ce786423db2e2f5c6fd4abf78, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/A/cac157a8d0be4d8294b608033637571f, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/A/f8119bc32b434411a0805be444b0a7fb, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/A/f9452fe87d2d47cebf1c15266ad1b782, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/A/6484aa56d8e04365921fd49f19d2f396, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/A/c72a8c34180b4c9d848eb6b85e6acca5, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/A/24337a4beed3414c9d6c55e64ad0b12a, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/A/25c3e3b7591a4e6da6c39ea3436974c9, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/A/77bb042f669d44bdbe1357482358c8ad] to archive 2024-11-18T20:14:00,799 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c0a89b2656d4:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-18T20:14:00,801 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c0a89b2656d4:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/A/13725551bd2245be8850dcc8a4559260 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/A/13725551bd2245be8850dcc8a4559260 2024-11-18T20:14:00,802 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c0a89b2656d4:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/A/882382ffd1fc46d2ab2a16547d84b48b to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/A/882382ffd1fc46d2ab2a16547d84b48b 2024-11-18T20:14:00,804 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c0a89b2656d4:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/A/6ea9a7c8ae2d4356a5bad1fa37d0ad60 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/A/6ea9a7c8ae2d4356a5bad1fa37d0ad60 2024-11-18T20:14:00,805 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c0a89b2656d4:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/A/bb7a6318ae714a60b837601adf470003 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/A/bb7a6318ae714a60b837601adf470003 2024-11-18T20:14:00,806 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c0a89b2656d4:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/A/98343979eecd4758af30dcd70afe64f5 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/A/98343979eecd4758af30dcd70afe64f5 2024-11-18T20:14:00,808 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c0a89b2656d4:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/A/c2eee0845c9c4513bd7be1efb6304d9e to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/A/c2eee0845c9c4513bd7be1efb6304d9e 2024-11-18T20:14:00,809 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c0a89b2656d4:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/A/98e854168bb1409588a759becb8a64d9 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/A/98e854168bb1409588a759becb8a64d9 2024-11-18T20:14:00,810 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c0a89b2656d4:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/A/95ce812f00ed4f3cbbdfb0528d61f7d1 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/A/95ce812f00ed4f3cbbdfb0528d61f7d1 2024-11-18T20:14:00,811 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c0a89b2656d4:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/A/586b29f729354327b15ca11a6051cc77 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/A/586b29f729354327b15ca11a6051cc77 2024-11-18T20:14:00,812 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c0a89b2656d4:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/A/9359688205b24faf89fd3d55818f214f to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/A/9359688205b24faf89fd3d55818f214f 2024-11-18T20:14:00,813 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c0a89b2656d4:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/A/b378673a59534d59b53e09a3fa6b7c48 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/A/b378673a59534d59b53e09a3fa6b7c48 2024-11-18T20:14:00,814 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c0a89b2656d4:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/A/a2fd1f59874841f984a78cecf53ebc9c to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/A/a2fd1f59874841f984a78cecf53ebc9c 2024-11-18T20:14:00,815 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c0a89b2656d4:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/A/1cefe7769199475883e4e4971a63f26c to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/A/1cefe7769199475883e4e4971a63f26c 2024-11-18T20:14:00,816 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c0a89b2656d4:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/A/d300b651488647d6bf3c7662bb435e6d to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/A/d300b651488647d6bf3c7662bb435e6d 2024-11-18T20:14:00,817 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c0a89b2656d4:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/A/6eb81deeae224aa496c3e80fd807719c to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/A/6eb81deeae224aa496c3e80fd807719c 2024-11-18T20:14:00,818 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c0a89b2656d4:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/A/e53f36ebfc4c42ecac30d3885f453e36 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/A/e53f36ebfc4c42ecac30d3885f453e36 2024-11-18T20:14:00,819 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c0a89b2656d4:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/A/cf8d6b0ce786423db2e2f5c6fd4abf78 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/A/cf8d6b0ce786423db2e2f5c6fd4abf78 2024-11-18T20:14:00,820 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c0a89b2656d4:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/A/cac157a8d0be4d8294b608033637571f to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/A/cac157a8d0be4d8294b608033637571f 2024-11-18T20:14:00,821 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c0a89b2656d4:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/A/f8119bc32b434411a0805be444b0a7fb to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/A/f8119bc32b434411a0805be444b0a7fb 2024-11-18T20:14:00,822 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c0a89b2656d4:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/A/f9452fe87d2d47cebf1c15266ad1b782 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/A/f9452fe87d2d47cebf1c15266ad1b782 2024-11-18T20:14:00,823 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c0a89b2656d4:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/A/6484aa56d8e04365921fd49f19d2f396 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/A/6484aa56d8e04365921fd49f19d2f396 2024-11-18T20:14:00,823 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c0a89b2656d4:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/A/c72a8c34180b4c9d848eb6b85e6acca5 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/A/c72a8c34180b4c9d848eb6b85e6acca5 2024-11-18T20:14:00,824 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c0a89b2656d4:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/A/24337a4beed3414c9d6c55e64ad0b12a to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/A/24337a4beed3414c9d6c55e64ad0b12a 2024-11-18T20:14:00,825 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c0a89b2656d4:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/A/25c3e3b7591a4e6da6c39ea3436974c9 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/A/25c3e3b7591a4e6da6c39ea3436974c9 2024-11-18T20:14:00,826 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c0a89b2656d4:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/A/77bb042f669d44bdbe1357482358c8ad to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/A/77bb042f669d44bdbe1357482358c8ad 2024-11-18T20:14:00,830 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c0a89b2656d4:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/B/cbd2cfe931ca46bab1044f305698551b, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/B/54484e6ef51b44bd802d7c46a02134e9, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/B/801435fe45ef41bba0024e6545b3ed57, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/B/3c7ab62230bf47c6b8d798c074e521e9, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/B/af3a6dea59e543c1991942d24c56605f, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/B/a448cbc67a664c2e9fec5ec7bf835c60, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/B/2e3e7c6ea78646e49f09afff72190a85, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/B/0a6c66df5a7b447daec65d9693b6176b, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/B/8ea784f2d434473fb944939472970ad3, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/B/33b13160ffcb4874ba770969fc043ef2, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/B/279c46f2a62d4ff4862d72bf147d5229, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/B/2514fc1351684a2d8e4ac89d24247572, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/B/2c5c4e80bf194e99a8386b7b5e720ba5, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/B/2cce7795a653443baa5ed2a7a2266e09, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/B/44f83273953d48f7b1d2dc70fa334c21, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/B/b056b619409e4fa080cfd13b3b919ad3, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/B/b96d30aaaee34e3f8b4c79d610182434, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/B/ded377de394f42e5a167b0ca1009ddac, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/B/967221e31fa3415f8d378c7fcdf65b68, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/B/2ffa04d7665a4b00837ccf092d337a3a, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/B/d418fff74b2d41e9a669c3e6aea22928, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/B/22d80fbc07a94cdba06fe23fcefd457d, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/B/9475a47b2d8943aea0759f94dc2ccdfb, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/B/37dfa2010f574c718cf52aba6b8ff150, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/B/2ae04bbc55d54d728bc6e782196448a8] to archive 2024-11-18T20:14:00,831 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c0a89b2656d4:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-18T20:14:00,832 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c0a89b2656d4:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/B/cbd2cfe931ca46bab1044f305698551b to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/B/cbd2cfe931ca46bab1044f305698551b 2024-11-18T20:14:00,833 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c0a89b2656d4:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/B/54484e6ef51b44bd802d7c46a02134e9 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/B/54484e6ef51b44bd802d7c46a02134e9 2024-11-18T20:14:00,834 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c0a89b2656d4:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/B/801435fe45ef41bba0024e6545b3ed57 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/B/801435fe45ef41bba0024e6545b3ed57 2024-11-18T20:14:00,835 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c0a89b2656d4:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/B/3c7ab62230bf47c6b8d798c074e521e9 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/B/3c7ab62230bf47c6b8d798c074e521e9 2024-11-18T20:14:00,836 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c0a89b2656d4:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/B/af3a6dea59e543c1991942d24c56605f to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/B/af3a6dea59e543c1991942d24c56605f 2024-11-18T20:14:00,836 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c0a89b2656d4:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/B/a448cbc67a664c2e9fec5ec7bf835c60 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/B/a448cbc67a664c2e9fec5ec7bf835c60 2024-11-18T20:14:00,837 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c0a89b2656d4:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/B/2e3e7c6ea78646e49f09afff72190a85 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/B/2e3e7c6ea78646e49f09afff72190a85 2024-11-18T20:14:00,838 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c0a89b2656d4:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/B/0a6c66df5a7b447daec65d9693b6176b to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/B/0a6c66df5a7b447daec65d9693b6176b 2024-11-18T20:14:00,839 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c0a89b2656d4:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/B/8ea784f2d434473fb944939472970ad3 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/B/8ea784f2d434473fb944939472970ad3 2024-11-18T20:14:00,839 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c0a89b2656d4:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/B/33b13160ffcb4874ba770969fc043ef2 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/B/33b13160ffcb4874ba770969fc043ef2 2024-11-18T20:14:00,840 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c0a89b2656d4:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/B/279c46f2a62d4ff4862d72bf147d5229 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/B/279c46f2a62d4ff4862d72bf147d5229 2024-11-18T20:14:00,841 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c0a89b2656d4:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/B/2514fc1351684a2d8e4ac89d24247572 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/B/2514fc1351684a2d8e4ac89d24247572 2024-11-18T20:14:00,841 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c0a89b2656d4:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/B/2c5c4e80bf194e99a8386b7b5e720ba5 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/B/2c5c4e80bf194e99a8386b7b5e720ba5 2024-11-18T20:14:00,842 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c0a89b2656d4:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/B/2cce7795a653443baa5ed2a7a2266e09 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/B/2cce7795a653443baa5ed2a7a2266e09 2024-11-18T20:14:00,843 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c0a89b2656d4:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/B/44f83273953d48f7b1d2dc70fa334c21 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/B/44f83273953d48f7b1d2dc70fa334c21 2024-11-18T20:14:00,844 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c0a89b2656d4:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/B/b056b619409e4fa080cfd13b3b919ad3 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/B/b056b619409e4fa080cfd13b3b919ad3 2024-11-18T20:14:00,845 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c0a89b2656d4:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/B/b96d30aaaee34e3f8b4c79d610182434 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/B/b96d30aaaee34e3f8b4c79d610182434 2024-11-18T20:14:00,846 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c0a89b2656d4:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/B/ded377de394f42e5a167b0ca1009ddac to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/B/ded377de394f42e5a167b0ca1009ddac 2024-11-18T20:14:00,847 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c0a89b2656d4:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/B/967221e31fa3415f8d378c7fcdf65b68 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/B/967221e31fa3415f8d378c7fcdf65b68 2024-11-18T20:14:00,847 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c0a89b2656d4:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/B/2ffa04d7665a4b00837ccf092d337a3a to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/B/2ffa04d7665a4b00837ccf092d337a3a 2024-11-18T20:14:00,848 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c0a89b2656d4:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/B/d418fff74b2d41e9a669c3e6aea22928 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/B/d418fff74b2d41e9a669c3e6aea22928 2024-11-18T20:14:00,849 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c0a89b2656d4:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/B/22d80fbc07a94cdba06fe23fcefd457d to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/B/22d80fbc07a94cdba06fe23fcefd457d 2024-11-18T20:14:00,850 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c0a89b2656d4:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/B/9475a47b2d8943aea0759f94dc2ccdfb to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/B/9475a47b2d8943aea0759f94dc2ccdfb 2024-11-18T20:14:00,851 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c0a89b2656d4:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/B/37dfa2010f574c718cf52aba6b8ff150 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/B/37dfa2010f574c718cf52aba6b8ff150 2024-11-18T20:14:00,852 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c0a89b2656d4:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/B/2ae04bbc55d54d728bc6e782196448a8 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/B/2ae04bbc55d54d728bc6e782196448a8 2024-11-18T20:14:00,855 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c0a89b2656d4:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/C/39768566c03c4227ad4ea16b60a92a68, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/C/48f3536a541f42668e7b5a2fbd8164ff, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/C/302b13df853b4d1ca57a92d03443373f, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/C/04e10e915057455dbe4a221d3d1ecbf2, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/C/8922951d492d486c9f2c499a300ecedc, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/C/3c284c0c7bb54f7086a229dab2f25984, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/C/488e247322a34eb9a7675567f725bc26, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/C/5910ecbcd40949fd8e61cad60c7f0aeb, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/C/130ab60e382345f4afd0d49dfa7d11c0, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/C/184d7265aa78461ba0f8d65d5af1633b, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/C/67b001ee8443430dbb148881a5bf85df, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/C/bd4b35b2165e4c7c8b671a04fe08f5da, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/C/941c6ba825684c128518e0753f084de3, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/C/32f4a989f1f9479ea7d0300d4fae626a, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/C/c2150536f53a48afb12ba0bbcef65e30, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/C/c235d8d94536475a9d237e3f0a65b6af, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/C/3e4de34e127544cba9c7b19533d00c3f, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/C/728d736b72a941109de410de2532a12f, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/C/c52d6bba54444cd3a22c51c225b0c781, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/C/26e9513e9840456689d3993329e2b085, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/C/9f3cdf7ab96c472a9ab3566c3a7dc863, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/C/470c305ee02d4575ae185191d19be0f4, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/C/1800cacd2c4b42b4ae32e3351cd8d1cb, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/C/739dfd966d6844699a10381a0fca4a7d, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/C/c34255e9dac2474e9fe6d79ce34b8221] to archive 2024-11-18T20:14:00,856 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c0a89b2656d4:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-18T20:14:00,858 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c0a89b2656d4:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/C/39768566c03c4227ad4ea16b60a92a68 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/C/39768566c03c4227ad4ea16b60a92a68 2024-11-18T20:14:00,859 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c0a89b2656d4:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/C/48f3536a541f42668e7b5a2fbd8164ff to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/C/48f3536a541f42668e7b5a2fbd8164ff 2024-11-18T20:14:00,859 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c0a89b2656d4:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/C/302b13df853b4d1ca57a92d03443373f to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/C/302b13df853b4d1ca57a92d03443373f 2024-11-18T20:14:00,860 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c0a89b2656d4:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/C/04e10e915057455dbe4a221d3d1ecbf2 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/C/04e10e915057455dbe4a221d3d1ecbf2 2024-11-18T20:14:00,861 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c0a89b2656d4:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/C/8922951d492d486c9f2c499a300ecedc to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/C/8922951d492d486c9f2c499a300ecedc 2024-11-18T20:14:00,862 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c0a89b2656d4:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/C/3c284c0c7bb54f7086a229dab2f25984 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/C/3c284c0c7bb54f7086a229dab2f25984 2024-11-18T20:14:00,863 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c0a89b2656d4:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/C/488e247322a34eb9a7675567f725bc26 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/C/488e247322a34eb9a7675567f725bc26 2024-11-18T20:14:00,863 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c0a89b2656d4:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/C/5910ecbcd40949fd8e61cad60c7f0aeb to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/C/5910ecbcd40949fd8e61cad60c7f0aeb 2024-11-18T20:14:00,864 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c0a89b2656d4:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/C/130ab60e382345f4afd0d49dfa7d11c0 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/C/130ab60e382345f4afd0d49dfa7d11c0 2024-11-18T20:14:00,865 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c0a89b2656d4:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/C/184d7265aa78461ba0f8d65d5af1633b to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/C/184d7265aa78461ba0f8d65d5af1633b 2024-11-18T20:14:00,866 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c0a89b2656d4:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/C/67b001ee8443430dbb148881a5bf85df to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/C/67b001ee8443430dbb148881a5bf85df 2024-11-18T20:14:00,867 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c0a89b2656d4:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/C/bd4b35b2165e4c7c8b671a04fe08f5da to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/C/bd4b35b2165e4c7c8b671a04fe08f5da 2024-11-18T20:14:00,867 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c0a89b2656d4:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/C/941c6ba825684c128518e0753f084de3 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/C/941c6ba825684c128518e0753f084de3 2024-11-18T20:14:00,868 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c0a89b2656d4:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/C/32f4a989f1f9479ea7d0300d4fae626a to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/C/32f4a989f1f9479ea7d0300d4fae626a 2024-11-18T20:14:00,869 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c0a89b2656d4:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/C/c2150536f53a48afb12ba0bbcef65e30 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/C/c2150536f53a48afb12ba0bbcef65e30 2024-11-18T20:14:00,870 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c0a89b2656d4:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/C/c235d8d94536475a9d237e3f0a65b6af to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/C/c235d8d94536475a9d237e3f0a65b6af 2024-11-18T20:14:00,871 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c0a89b2656d4:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/C/3e4de34e127544cba9c7b19533d00c3f to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/C/3e4de34e127544cba9c7b19533d00c3f 2024-11-18T20:14:00,872 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c0a89b2656d4:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/C/728d736b72a941109de410de2532a12f to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/C/728d736b72a941109de410de2532a12f 2024-11-18T20:14:00,872 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c0a89b2656d4:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/C/c52d6bba54444cd3a22c51c225b0c781 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/C/c52d6bba54444cd3a22c51c225b0c781 2024-11-18T20:14:00,873 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c0a89b2656d4:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/C/26e9513e9840456689d3993329e2b085 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/C/26e9513e9840456689d3993329e2b085 2024-11-18T20:14:00,874 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c0a89b2656d4:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/C/9f3cdf7ab96c472a9ab3566c3a7dc863 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/C/9f3cdf7ab96c472a9ab3566c3a7dc863 2024-11-18T20:14:00,875 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c0a89b2656d4:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/C/470c305ee02d4575ae185191d19be0f4 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/C/470c305ee02d4575ae185191d19be0f4 2024-11-18T20:14:00,875 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c0a89b2656d4:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/C/1800cacd2c4b42b4ae32e3351cd8d1cb to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/C/1800cacd2c4b42b4ae32e3351cd8d1cb 2024-11-18T20:14:00,876 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c0a89b2656d4:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/C/739dfd966d6844699a10381a0fca4a7d to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/C/739dfd966d6844699a10381a0fca4a7d 2024-11-18T20:14:00,877 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c0a89b2656d4:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/C/c34255e9dac2474e9fe6d79ce34b8221 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/C/c34255e9dac2474e9fe6d79ce34b8221 2024-11-18T20:14:01,019 INFO [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=364, memsize=2.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/.tmp/A/6a64ab9e2f9f435d90ed15fc18325943 2024-11-18T20:14:01,026 DEBUG [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/.tmp/B/8d316d07f73f4acab31ea090711aa6f7 is 50, key is test_row_2/B:col10/1731960838503/Put/seqid=0 2024-11-18T20:14:01,029 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742280_1456 (size=7415) 2024-11-18T20:14:01,430 INFO [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=2.24 KB at sequenceid=364 (bloomFilter=true), to=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/.tmp/B/8d316d07f73f4acab31ea090711aa6f7 2024-11-18T20:14:01,435 DEBUG [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/.tmp/C/9a26c54d26d84e958b647372dccb57a6 is 50, key is test_row_2/C:col10/1731960838503/Put/seqid=0 2024-11-18T20:14:01,438 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742281_1457 (size=7415) 2024-11-18T20:14:01,839 INFO [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=2.24 KB at sequenceid=364 (bloomFilter=true), to=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/.tmp/C/9a26c54d26d84e958b647372dccb57a6 2024-11-18T20:14:01,843 DEBUG [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/.tmp/A/6a64ab9e2f9f435d90ed15fc18325943 as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/A/6a64ab9e2f9f435d90ed15fc18325943 2024-11-18T20:14:01,847 INFO [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/A/6a64ab9e2f9f435d90ed15fc18325943, entries=50, sequenceid=364, filesize=13.5 K 2024-11-18T20:14:01,847 DEBUG [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/.tmp/B/8d316d07f73f4acab31ea090711aa6f7 as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/B/8d316d07f73f4acab31ea090711aa6f7 2024-11-18T20:14:01,851 INFO [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/B/8d316d07f73f4acab31ea090711aa6f7, entries=50, sequenceid=364, filesize=7.2 K 2024-11-18T20:14:01,852 DEBUG [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/.tmp/C/9a26c54d26d84e958b647372dccb57a6 as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/C/9a26c54d26d84e958b647372dccb57a6 2024-11-18T20:14:01,856 INFO [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/C/9a26c54d26d84e958b647372dccb57a6, entries=50, sequenceid=364, filesize=7.2 K 2024-11-18T20:14:01,857 INFO [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HRegion(3140): Finished flush of dataSize ~6.71 KB/6870, heapSize ~18.28 KB/18720, currentSize=0 B/0 for 8eaa77b8f4c3de4047bf5415370077cd in 1659ms, sequenceid=364, compaction requested=true 2024-11-18T20:14:01,862 DEBUG [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/recovered.edits/367.seqid, newMaxSeqId=367, maxSeqId=4 2024-11-18T20:14:01,862 INFO [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd. 2024-11-18T20:14:01,863 DEBUG [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HRegion(1676): Region close journal for 8eaa77b8f4c3de4047bf5415370077cd: Waiting for close lock at 1731960838672Running coprocessor pre-close hooks at 1731960838672Disabling compacts and flushes for region at 1731960838672Disabling writes for close at 1731960840198 (+1526 ms)Obtaining lock to block concurrent updates at 1731960840198Preparing flush snapshotting stores in 8eaa77b8f4c3de4047bf5415370077cd at 1731960840198Finished memstore snapshotting TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd., syncing WAL and waiting on mvcc, flushsize=dataSize=6870, getHeapSize=18720, getOffHeapSize=0, getCellsCount=150 at 1731960840198Flushing stores of TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd. at 1731960840199 (+1 ms)Flushing 8eaa77b8f4c3de4047bf5415370077cd/A: creating writer at 1731960840199Flushing mob file 8eaa77b8f4c3de4047bf5415370077cd/A: appending metadata at 1731960840204 (+5 ms)Flushing mob file 8eaa77b8f4c3de4047bf5415370077cd/A: closing flushed file at 1731960840204Flushing 8eaa77b8f4c3de4047bf5415370077cd/A: appending metadata at 1731960840614 (+410 ms)Flushing 8eaa77b8f4c3de4047bf5415370077cd/A: closing flushed file at 1731960840615 (+1 ms)Flushing 8eaa77b8f4c3de4047bf5415370077cd/B: creating writer at 1731960841023 (+408 ms)Flushing 8eaa77b8f4c3de4047bf5415370077cd/B: appending metadata at 1731960841025 (+2 ms)Flushing 8eaa77b8f4c3de4047bf5415370077cd/B: closing flushed file at 1731960841025Flushing 8eaa77b8f4c3de4047bf5415370077cd/C: creating writer at 1731960841433 (+408 ms)Flushing 8eaa77b8f4c3de4047bf5415370077cd/C: appending metadata at 1731960841435 (+2 ms)Flushing 8eaa77b8f4c3de4047bf5415370077cd/C: closing flushed file at 1731960841435Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@706cce3a: reopening flushed file at 1731960841842 (+407 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@10e7b787: reopening flushed file at 1731960841847 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2e4b4490: reopening flushed file at 1731960841851 (+4 ms)Finished flush of dataSize ~6.71 KB/6870, heapSize ~18.28 KB/18720, currentSize=0 B/0 for 8eaa77b8f4c3de4047bf5415370077cd in 1659ms, sequenceid=364, compaction requested=true at 1731960841857 (+6 ms)Writing region close event to WAL at 1731960841859 (+2 ms)Running coprocessor post-close hooks at 1731960841862 (+3 ms)Closed at 1731960841862 2024-11-18T20:14:01,865 INFO [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] handler.UnassignRegionHandler(157): Closed 8eaa77b8f4c3de4047bf5415370077cd 2024-11-18T20:14:01,865 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=115 updating hbase:meta row=8eaa77b8f4c3de4047bf5415370077cd, regionState=CLOSED 2024-11-18T20:14:01,867 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=116, ppid=115, state=RUNNABLE, hasLock=false; CloseRegionProcedure 8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 because future has completed 2024-11-18T20:14:01,870 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=116, resume processing ppid=115 2024-11-18T20:14:01,870 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=116, ppid=115, state=SUCCESS, hasLock=false; CloseRegionProcedure 8eaa77b8f4c3de4047bf5415370077cd, server=c0a89b2656d4,35963,1731960718223 in 3.3480 sec 2024-11-18T20:14:01,877 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=115, resume processing ppid=114 2024-11-18T20:14:01,877 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=115, ppid=114, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=8eaa77b8f4c3de4047bf5415370077cd, UNASSIGN in 3.3540 sec 2024-11-18T20:14:01,878 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=114, resume processing ppid=113 2024-11-18T20:14:01,878 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=114, ppid=113, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 3.3620 sec 2024-11-18T20:14:01,880 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731960841879"}]},"ts":"1731960841879"} 2024-11-18T20:14:01,881 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-18T20:14:01,881 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-18T20:14:01,884 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=113, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 3.3740 sec 2024-11-18T20:14:02,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-18T20:14:02,674 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-18T20:14:02,674 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-18T20:14:02,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] procedure2.ProcedureExecutor(1139): Stored pid=117, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-18T20:14:02,677 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=117, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-18T20:14:02,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-18T20:14:02,678 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=117, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-18T20:14:02,680 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd 2024-11-18T20:14:02,684 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/A, FileablePath, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/B, FileablePath, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/C, FileablePath, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/recovered.edits] 2024-11-18T20:14:02,687 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/A/239f37fb9491422f989ff9785689a834 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/A/239f37fb9491422f989ff9785689a834 2024-11-18T20:14:02,689 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/A/6a64ab9e2f9f435d90ed15fc18325943 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/A/6a64ab9e2f9f435d90ed15fc18325943 2024-11-18T20:14:02,691 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/A/c3241a38fc354bdf8ba9791b40d6366f to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/A/c3241a38fc354bdf8ba9791b40d6366f 2024-11-18T20:14:02,695 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/B/044f39110875474aa77bd3a600c19d5b to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/B/044f39110875474aa77bd3a600c19d5b 2024-11-18T20:14:02,696 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/B/1feca162e4a14994bfa54671cdd8583c to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/B/1feca162e4a14994bfa54671cdd8583c 2024-11-18T20:14:02,697 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/B/8d316d07f73f4acab31ea090711aa6f7 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/B/8d316d07f73f4acab31ea090711aa6f7 2024-11-18T20:14:02,698 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/C/0465edff8efa4236a100e313b8715862 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/C/0465edff8efa4236a100e313b8715862 2024-11-18T20:14:02,699 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/C/3e6a00ad32094ecca8236623cdcf3d9b to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/C/3e6a00ad32094ecca8236623cdcf3d9b 2024-11-18T20:14:02,700 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/C/9a26c54d26d84e958b647372dccb57a6 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/C/9a26c54d26d84e958b647372dccb57a6 2024-11-18T20:14:02,701 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/recovered.edits/367.seqid to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd/recovered.edits/367.seqid 2024-11-18T20:14:02,701 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/8eaa77b8f4c3de4047bf5415370077cd 2024-11-18T20:14:02,702 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-18T20:14:02,702 DEBUG [PEWorker-5 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-18T20:14:02,702 DEBUG [PEWorker-5 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-18T20:14:02,704 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118092c6c9cedef4d28b7f50dbd4928f09e_8eaa77b8f4c3de4047bf5415370077cd to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118092c6c9cedef4d28b7f50dbd4928f09e_8eaa77b8f4c3de4047bf5415370077cd 2024-11-18T20:14:02,705 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111810150afb837f4a3db3408cd03f824a61_8eaa77b8f4c3de4047bf5415370077cd to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111810150afb837f4a3db3408cd03f824a61_8eaa77b8f4c3de4047bf5415370077cd 2024-11-18T20:14:02,706 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118193793844f7a4b9fb5e484d2ce5540f6_8eaa77b8f4c3de4047bf5415370077cd to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118193793844f7a4b9fb5e484d2ce5540f6_8eaa77b8f4c3de4047bf5415370077cd 2024-11-18T20:14:02,707 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411181a87ceb9cd434ddf8ac4006d2e920eb9_8eaa77b8f4c3de4047bf5415370077cd to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411181a87ceb9cd434ddf8ac4006d2e920eb9_8eaa77b8f4c3de4047bf5415370077cd 2024-11-18T20:14:02,707 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111833db8552e4124ed695a73efe62c6b815_8eaa77b8f4c3de4047bf5415370077cd to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111833db8552e4124ed695a73efe62c6b815_8eaa77b8f4c3de4047bf5415370077cd 2024-11-18T20:14:02,708 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411183a7dacb0269b4a3182554f48e6dfb16c_8eaa77b8f4c3de4047bf5415370077cd to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411183a7dacb0269b4a3182554f48e6dfb16c_8eaa77b8f4c3de4047bf5415370077cd 2024-11-18T20:14:02,709 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411183fbdfe8f0fb848ce8cdb98216cbb6efe_8eaa77b8f4c3de4047bf5415370077cd to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411183fbdfe8f0fb848ce8cdb98216cbb6efe_8eaa77b8f4c3de4047bf5415370077cd 2024-11-18T20:14:02,709 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411188581e71857c14365b7d6c279a6714438_8eaa77b8f4c3de4047bf5415370077cd to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411188581e71857c14365b7d6c279a6714438_8eaa77b8f4c3de4047bf5415370077cd 2024-11-18T20:14:02,710 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111888c747970e7e4f3c9628cb9a14220339_8eaa77b8f4c3de4047bf5415370077cd to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111888c747970e7e4f3c9628cb9a14220339_8eaa77b8f4c3de4047bf5415370077cd 2024-11-18T20:14:02,711 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118945f9a1dad114c1abd2374b09f1040e5_8eaa77b8f4c3de4047bf5415370077cd to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118945f9a1dad114c1abd2374b09f1040e5_8eaa77b8f4c3de4047bf5415370077cd 2024-11-18T20:14:02,711 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411189fa2a4ec64054691ae6ab627b13e4ab7_8eaa77b8f4c3de4047bf5415370077cd to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411189fa2a4ec64054691ae6ab627b13e4ab7_8eaa77b8f4c3de4047bf5415370077cd 2024-11-18T20:14:02,712 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118a0e3cc7a8e6c404f954ba12bc50f5dbd_8eaa77b8f4c3de4047bf5415370077cd to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118a0e3cc7a8e6c404f954ba12bc50f5dbd_8eaa77b8f4c3de4047bf5415370077cd 2024-11-18T20:14:02,713 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118a28a3f9035c24184a43a33d30e7ecefb_8eaa77b8f4c3de4047bf5415370077cd to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118a28a3f9035c24184a43a33d30e7ecefb_8eaa77b8f4c3de4047bf5415370077cd 2024-11-18T20:14:02,714 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118b082c61d9f9e49739f43f197b942b9a5_8eaa77b8f4c3de4047bf5415370077cd to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118b082c61d9f9e49739f43f197b942b9a5_8eaa77b8f4c3de4047bf5415370077cd 2024-11-18T20:14:02,714 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118bbf6203c7ab24b2ea960ea16a99f1e5b_8eaa77b8f4c3de4047bf5415370077cd to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118bbf6203c7ab24b2ea960ea16a99f1e5b_8eaa77b8f4c3de4047bf5415370077cd 2024-11-18T20:14:02,715 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118bf411a600a6f4d5b945aae711923a324_8eaa77b8f4c3de4047bf5415370077cd to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118bf411a600a6f4d5b945aae711923a324_8eaa77b8f4c3de4047bf5415370077cd 2024-11-18T20:14:02,715 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118e0229bb8b17b48c99770f1dda2191c5e_8eaa77b8f4c3de4047bf5415370077cd to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118e0229bb8b17b48c99770f1dda2191c5e_8eaa77b8f4c3de4047bf5415370077cd 2024-11-18T20:14:02,716 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118ea198b77737e422abb6b17be84f80a94_8eaa77b8f4c3de4047bf5415370077cd to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118ea198b77737e422abb6b17be84f80a94_8eaa77b8f4c3de4047bf5415370077cd 2024-11-18T20:14:02,717 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118eadf2ac8ee30428f8e305613dbf7db51_8eaa77b8f4c3de4047bf5415370077cd to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118eadf2ac8ee30428f8e305613dbf7db51_8eaa77b8f4c3de4047bf5415370077cd 2024-11-18T20:14:02,718 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118f8d1fa9da01c46999a59c298aeccd180_8eaa77b8f4c3de4047bf5415370077cd to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118f8d1fa9da01c46999a59c298aeccd180_8eaa77b8f4c3de4047bf5415370077cd 2024-11-18T20:14:02,718 DEBUG [PEWorker-5 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-18T20:14:02,720 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=117, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-18T20:14:02,722 WARN [PEWorker-5 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-18T20:14:02,723 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-18T20:14:02,725 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=117, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-18T20:14:02,725 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-18T20:14:02,725 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731960842725"}]},"ts":"9223372036854775807"} 2024-11-18T20:14:02,727 INFO [PEWorker-5 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-18T20:14:02,727 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 8eaa77b8f4c3de4047bf5415370077cd, NAME => 'TestAcidGuarantees,,1731960806870.8eaa77b8f4c3de4047bf5415370077cd.', STARTKEY => '', ENDKEY => ''}] 2024-11-18T20:14:02,727 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-18T20:14:02,727 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731960842727"}]},"ts":"9223372036854775807"} 2024-11-18T20:14:02,729 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-18T20:14:02,729 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(133): Finished pid=117, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-18T20:14:02,731 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=117, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 55 msec 2024-11-18T20:14:02,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-18T20:14:02,784 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-18T20:14:02,787 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-18T20:14:02,797 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testMobScanAtomicity Thread=237 (was 239), OpenFileDescriptor=445 (was 453), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=454 (was 513), ProcessCount=11 (was 11), AvailableMemoryMB=2493 (was 2580) 2024-11-18T20:14:02,807 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testScanAtomicity Thread=237, OpenFileDescriptor=445, MaxFileDescriptor=1048576, SystemLoadAverage=454, ProcessCount=11, AvailableMemoryMB=2493 2024-11-18T20:14:02,808 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-18T20:14:02,808 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-18T20:14:02,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] procedure2.ProcedureExecutor(1139): Stored pid=118, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-18T20:14:02,811 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=118, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-18T20:14:02,811 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:02,811 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 118 2024-11-18T20:14:02,812 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=118, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-18T20:14:02,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=118 2024-11-18T20:14:02,816 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742282_1458 (size=959) 2024-11-18T20:14:02,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=118 2024-11-18T20:14:03,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=118 2024-11-18T20:14:03,222 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 23d90021d1eda01703e7bdf7270f74fc, NAME => 'TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123 2024-11-18T20:14:03,229 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742283_1459 (size=53) 2024-11-18T20:14:03,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=118 2024-11-18T20:14:03,632 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T20:14:03,632 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 23d90021d1eda01703e7bdf7270f74fc, disabling compactions & flushes 2024-11-18T20:14:03,632 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc. 2024-11-18T20:14:03,633 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc. 2024-11-18T20:14:03,633 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc. after waiting 0 ms 2024-11-18T20:14:03,633 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc. 2024-11-18T20:14:03,633 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc. 2024-11-18T20:14:03,633 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 23d90021d1eda01703e7bdf7270f74fc: Waiting for close lock at 1731960843632Disabling compacts and flushes for region at 1731960843632Disabling writes for close at 1731960843633 (+1 ms)Writing region close event to WAL at 1731960843633Closed at 1731960843633 2024-11-18T20:14:03,635 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=118, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-18T20:14:03,636 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731960843635"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731960843635"}]},"ts":"1731960843635"} 2024-11-18T20:14:03,639 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-18T20:14:03,640 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=118, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-18T20:14:03,641 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731960843640"}]},"ts":"1731960843640"} 2024-11-18T20:14:03,642 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-18T20:14:03,643 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=119, ppid=118, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=23d90021d1eda01703e7bdf7270f74fc, ASSIGN}] 2024-11-18T20:14:03,644 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=119, ppid=118, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=23d90021d1eda01703e7bdf7270f74fc, ASSIGN 2024-11-18T20:14:03,645 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=119, ppid=118, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=23d90021d1eda01703e7bdf7270f74fc, ASSIGN; state=OFFLINE, location=c0a89b2656d4,35963,1731960718223; forceNewPlan=false, retain=false 2024-11-18T20:14:03,796 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=119 updating hbase:meta row=23d90021d1eda01703e7bdf7270f74fc, regionState=OPENING, regionLocation=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:03,798 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=119, ppid=118, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=23d90021d1eda01703e7bdf7270f74fc, ASSIGN because future has completed 2024-11-18T20:14:03,798 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=120, ppid=119, state=RUNNABLE, hasLock=false; OpenRegionProcedure 23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223}] 2024-11-18T20:14:03,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=118 2024-11-18T20:14:03,956 INFO [RS_OPEN_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc. 2024-11-18T20:14:03,957 DEBUG [RS_OPEN_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(7752): Opening region: {ENCODED => 23d90021d1eda01703e7bdf7270f74fc, NAME => 'TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc.', STARTKEY => '', ENDKEY => ''} 2024-11-18T20:14:03,958 DEBUG [RS_OPEN_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 23d90021d1eda01703e7bdf7270f74fc 2024-11-18T20:14:03,958 DEBUG [RS_OPEN_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T20:14:03,959 DEBUG [RS_OPEN_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(7794): checking encryption for 23d90021d1eda01703e7bdf7270f74fc 2024-11-18T20:14:03,959 DEBUG [RS_OPEN_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(7797): checking classloading for 23d90021d1eda01703e7bdf7270f74fc 2024-11-18T20:14:03,962 INFO [StoreOpener-23d90021d1eda01703e7bdf7270f74fc-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 23d90021d1eda01703e7bdf7270f74fc 2024-11-18T20:14:03,965 INFO [StoreOpener-23d90021d1eda01703e7bdf7270f74fc-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 23d90021d1eda01703e7bdf7270f74fc columnFamilyName A 2024-11-18T20:14:03,965 DEBUG [StoreOpener-23d90021d1eda01703e7bdf7270f74fc-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:03,965 INFO [StoreOpener-23d90021d1eda01703e7bdf7270f74fc-1 {}] regionserver.HStore(327): Store=23d90021d1eda01703e7bdf7270f74fc/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T20:14:03,965 INFO [StoreOpener-23d90021d1eda01703e7bdf7270f74fc-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 23d90021d1eda01703e7bdf7270f74fc 2024-11-18T20:14:03,966 INFO [StoreOpener-23d90021d1eda01703e7bdf7270f74fc-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 23d90021d1eda01703e7bdf7270f74fc columnFamilyName B 2024-11-18T20:14:03,966 DEBUG [StoreOpener-23d90021d1eda01703e7bdf7270f74fc-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:03,966 INFO [StoreOpener-23d90021d1eda01703e7bdf7270f74fc-1 {}] regionserver.HStore(327): Store=23d90021d1eda01703e7bdf7270f74fc/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T20:14:03,967 INFO [StoreOpener-23d90021d1eda01703e7bdf7270f74fc-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 23d90021d1eda01703e7bdf7270f74fc 2024-11-18T20:14:03,967 INFO [StoreOpener-23d90021d1eda01703e7bdf7270f74fc-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 23d90021d1eda01703e7bdf7270f74fc columnFamilyName C 2024-11-18T20:14:03,967 DEBUG [StoreOpener-23d90021d1eda01703e7bdf7270f74fc-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:03,968 INFO [StoreOpener-23d90021d1eda01703e7bdf7270f74fc-1 {}] regionserver.HStore(327): Store=23d90021d1eda01703e7bdf7270f74fc/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T20:14:03,968 DEBUG [RS_OPEN_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(1038): replaying wal for 23d90021d1eda01703e7bdf7270f74fc 2024-11-18T20:14:03,968 DEBUG [RS_OPEN_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc 2024-11-18T20:14:03,969 DEBUG [RS_OPEN_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc 2024-11-18T20:14:03,970 DEBUG [RS_OPEN_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(1048): stopping wal replay for 23d90021d1eda01703e7bdf7270f74fc 2024-11-18T20:14:03,970 DEBUG [RS_OPEN_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(1060): Cleaning up temporary data for 23d90021d1eda01703e7bdf7270f74fc 2024-11-18T20:14:03,970 DEBUG [RS_OPEN_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-18T20:14:03,971 DEBUG [RS_OPEN_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(1093): writing seq id for 23d90021d1eda01703e7bdf7270f74fc 2024-11-18T20:14:03,972 DEBUG [RS_OPEN_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-18T20:14:03,972 INFO [RS_OPEN_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(1114): Opened 23d90021d1eda01703e7bdf7270f74fc; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68620690, jitterRate=0.022527962923049927}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-18T20:14:03,972 DEBUG [RS_OPEN_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 23d90021d1eda01703e7bdf7270f74fc 2024-11-18T20:14:03,973 DEBUG [RS_OPEN_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(1006): Region open journal for 23d90021d1eda01703e7bdf7270f74fc: Running coprocessor pre-open hook at 1731960843959Writing region info on filesystem at 1731960843959Initializing all the Stores at 1731960843960 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731960843961 (+1 ms)Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731960843961Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731960843961Cleaning up temporary data from old regions at 1731960843970 (+9 ms)Running coprocessor post-open hooks at 1731960843972 (+2 ms)Region opened successfully at 1731960843973 (+1 ms) 2024-11-18T20:14:03,973 INFO [RS_OPEN_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., pid=120, masterSystemTime=1731960843951 2024-11-18T20:14:03,975 DEBUG [RS_OPEN_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc. 2024-11-18T20:14:03,975 INFO [RS_OPEN_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc. 2024-11-18T20:14:03,975 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=119 updating hbase:meta row=23d90021d1eda01703e7bdf7270f74fc, regionState=OPEN, openSeqNum=2, regionLocation=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:03,976 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=120, ppid=119, state=RUNNABLE, hasLock=false; OpenRegionProcedure 23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 because future has completed 2024-11-18T20:14:03,978 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=120, resume processing ppid=119 2024-11-18T20:14:03,979 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=120, ppid=119, state=SUCCESS, hasLock=false; OpenRegionProcedure 23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 in 179 msec 2024-11-18T20:14:03,980 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=119, resume processing ppid=118 2024-11-18T20:14:03,980 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=119, ppid=118, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=23d90021d1eda01703e7bdf7270f74fc, ASSIGN in 336 msec 2024-11-18T20:14:03,981 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=118, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-18T20:14:03,981 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731960843981"}]},"ts":"1731960843981"} 2024-11-18T20:14:03,982 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-18T20:14:03,982 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=118, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-18T20:14:03,984 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=118, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1750 sec 2024-11-18T20:14:04,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=118 2024-11-18T20:14:04,964 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-18T20:14:04,966 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5dc40216, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:14:04,966 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c0a89b2656d4,41277,-1 for getting cluster id 2024-11-18T20:14:04,966 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T20:14:04,968 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ac99df6b-7791-4693-a81a-6a5a39504490' 2024-11-18T20:14:04,968 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T20:14:04,968 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ac99df6b-7791-4693-a81a-6a5a39504490" 2024-11-18T20:14:04,969 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@77f0e67b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:14:04,969 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c0a89b2656d4,41277,-1] 2024-11-18T20:14:04,969 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T20:14:04,969 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:14:04,971 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60464, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T20:14:04,972 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1a888376, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:14:04,973 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T20:14:04,974 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c0a89b2656d4,35963,1731960718223, seqNum=-1] 2024-11-18T20:14:04,974 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T20:14:04,975 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51848, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T20:14:04,975 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-18T20:14:04,976 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncConnectionImpl(321): The fetched master address is c0a89b2656d4,41277,1731960717534 2024-11-18T20:14:04,976 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@18a8519d 2024-11-18T20:14:04,976 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-18T20:14:04,977 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60476, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-18T20:14:04,978 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@be1708d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:14:04,978 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c0a89b2656d4,41277,-1 for getting cluster id 2024-11-18T20:14:04,979 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T20:14:04,979 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ac99df6b-7791-4693-a81a-6a5a39504490' 2024-11-18T20:14:04,979 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T20:14:04,979 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ac99df6b-7791-4693-a81a-6a5a39504490" 2024-11-18T20:14:04,979 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@78629fe3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:14:04,980 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c0a89b2656d4,41277,-1] 2024-11-18T20:14:04,980 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T20:14:04,980 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:14:04,980 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60492, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T20:14:04,981 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1c88c2f5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:14:04,981 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6516064e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:14:04,981 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c0a89b2656d4,41277,-1 for getting cluster id 2024-11-18T20:14:04,982 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T20:14:04,982 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ac99df6b-7791-4693-a81a-6a5a39504490' 2024-11-18T20:14:04,982 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T20:14:04,982 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ac99df6b-7791-4693-a81a-6a5a39504490" 2024-11-18T20:14:04,983 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@10afc551, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:14:04,983 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c0a89b2656d4,41277,-1] 2024-11-18T20:14:04,983 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T20:14:04,983 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:14:04,984 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60508, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T20:14:04,984 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3295face, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:14:04,985 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@16fe0696, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:14:04,985 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c0a89b2656d4,41277,-1 for getting cluster id 2024-11-18T20:14:04,985 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T20:14:04,985 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ac99df6b-7791-4693-a81a-6a5a39504490' 2024-11-18T20:14:04,985 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T20:14:04,986 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ac99df6b-7791-4693-a81a-6a5a39504490" 2024-11-18T20:14:04,986 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@69307dc8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:14:04,986 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c0a89b2656d4,41277,-1] 2024-11-18T20:14:04,986 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T20:14:04,986 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:14:04,986 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60532, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T20:14:04,987 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@323be5ca, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:14:04,988 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@59de117f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:14:04,988 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c0a89b2656d4,41277,-1 for getting cluster id 2024-11-18T20:14:04,988 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T20:14:04,988 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ac99df6b-7791-4693-a81a-6a5a39504490' 2024-11-18T20:14:04,988 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T20:14:04,988 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ac99df6b-7791-4693-a81a-6a5a39504490" 2024-11-18T20:14:04,988 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3a78d279, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:14:04,988 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c0a89b2656d4,41277,-1] 2024-11-18T20:14:04,989 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T20:14:04,989 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:14:04,989 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60544, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T20:14:04,990 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3aa7142b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:14:04,990 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@472a8168, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:14:04,990 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c0a89b2656d4,41277,-1 for getting cluster id 2024-11-18T20:14:04,990 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T20:14:04,991 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ac99df6b-7791-4693-a81a-6a5a39504490' 2024-11-18T20:14:04,991 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T20:14:04,991 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ac99df6b-7791-4693-a81a-6a5a39504490" 2024-11-18T20:14:04,991 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4fd77491, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:14:04,991 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c0a89b2656d4,41277,-1] 2024-11-18T20:14:04,991 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T20:14:04,992 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:14:04,998 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60570, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T20:14:04,999 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@60e82f87, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:14:04,999 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5c296bf5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:14:04,999 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c0a89b2656d4,41277,-1 for getting cluster id 2024-11-18T20:14:05,000 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T20:14:05,000 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ac99df6b-7791-4693-a81a-6a5a39504490' 2024-11-18T20:14:05,000 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T20:14:05,000 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ac99df6b-7791-4693-a81a-6a5a39504490" 2024-11-18T20:14:05,000 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@72d7c1d1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:14:05,000 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c0a89b2656d4,41277,-1] 2024-11-18T20:14:05,001 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T20:14:05,001 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:14:05,001 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60588, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T20:14:05,002 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@79956e4b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:14:05,002 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@47bc1e91, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:14:05,002 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c0a89b2656d4,41277,-1 for getting cluster id 2024-11-18T20:14:05,002 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T20:14:05,003 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ac99df6b-7791-4693-a81a-6a5a39504490' 2024-11-18T20:14:05,003 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T20:14:05,003 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ac99df6b-7791-4693-a81a-6a5a39504490" 2024-11-18T20:14:05,003 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@513077d5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:14:05,003 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c0a89b2656d4,41277,-1] 2024-11-18T20:14:05,003 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T20:14:05,003 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:14:05,004 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60598, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T20:14:05,004 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1b6b3dd0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:14:05,005 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6bc5e64, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:14:05,005 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c0a89b2656d4,41277,-1 for getting cluster id 2024-11-18T20:14:05,005 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T20:14:05,005 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ac99df6b-7791-4693-a81a-6a5a39504490' 2024-11-18T20:14:05,005 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T20:14:05,005 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ac99df6b-7791-4693-a81a-6a5a39504490" 2024-11-18T20:14:05,006 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@755d5b1c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:14:05,006 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c0a89b2656d4,41277,-1] 2024-11-18T20:14:05,006 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T20:14:05,006 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:14:05,006 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60604, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T20:14:05,007 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@39d55c9b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:14:05,007 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@42daa6ab, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:14:05,007 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c0a89b2656d4,41277,-1 for getting cluster id 2024-11-18T20:14:05,007 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T20:14:05,008 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ac99df6b-7791-4693-a81a-6a5a39504490' 2024-11-18T20:14:05,008 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T20:14:05,008 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ac99df6b-7791-4693-a81a-6a5a39504490" 2024-11-18T20:14:05,008 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@135ae676, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:14:05,008 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c0a89b2656d4,41277,-1] 2024-11-18T20:14:05,008 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T20:14:05,009 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:14:05,009 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60624, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T20:14:05,009 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@46513ea4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:14:05,010 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@366fb241, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:14:05,010 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c0a89b2656d4,41277,-1 for getting cluster id 2024-11-18T20:14:05,010 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T20:14:05,011 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ac99df6b-7791-4693-a81a-6a5a39504490' 2024-11-18T20:14:05,011 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T20:14:05,011 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ac99df6b-7791-4693-a81a-6a5a39504490" 2024-11-18T20:14:05,011 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3dee4e1c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:14:05,011 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c0a89b2656d4,41277,-1] 2024-11-18T20:14:05,011 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T20:14:05,011 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:14:05,012 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60636, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T20:14:05,012 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@31b2420c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:14:05,013 DEBUG [Thread-2061 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T20:14:05,013 DEBUG [Thread-2055 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T20:14:05,013 DEBUG [Thread-2059 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T20:14:05,013 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c0a89b2656d4,35963,1731960718223, seqNum=-1] 2024-11-18T20:14:05,013 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c0a89b2656d4,35963,1731960718223, seqNum=-1] 2024-11-18T20:14:05,014 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T20:14:05,014 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T20:14:05,014 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-18T20:14:05,014 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c0a89b2656d4,35963,1731960718223, seqNum=-1] 2024-11-18T20:14:05,014 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T20:14:05,014 DEBUG [Thread-2068 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T20:14:05,014 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51860, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T20:14:05,014 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51878, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T20:14:05,015 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51862, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T20:14:05,015 DEBUG [Thread-2057 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T20:14:05,015 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c0a89b2656d4,35963,1731960718223, seqNum=-1] 2024-11-18T20:14:05,015 DEBUG [Thread-2070 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T20:14:05,015 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T20:14:05,015 DEBUG [Thread-2053 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T20:14:05,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] procedure2.ProcedureExecutor(1139): Stored pid=121, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees 2024-11-18T20:14:05,016 DEBUG [Thread-2072 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T20:14:05,016 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c0a89b2656d4,35963,1731960718223, seqNum=-1] 2024-11-18T20:14:05,016 DEBUG [Thread-2064 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T20:14:05,016 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T20:14:05,017 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c0a89b2656d4,35963,1731960718223, seqNum=-1] 2024-11-18T20:14:05,017 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2] 2024-11-18T20:14:05,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-18T20:14:05,017 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c0a89b2656d4,35963,1731960718223, seqNum=-1] 2024-11-18T20:14:05,017 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T20:14:05,017 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=121, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-18T20:14:05,017 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T20:14:05,017 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c0a89b2656d4,35963,1731960718223, seqNum=-1] 2024-11-18T20:14:05,017 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c0a89b2656d4,35963,1731960718223, seqNum=-1] 2024-11-18T20:14:05,017 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51888, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T20:14:05,017 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T20:14:05,017 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T20:14:05,018 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51894, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T20:14:05,018 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2] 2024-11-18T20:14:05,018 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2] 2024-11-18T20:14:05,019 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51912, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T20:14:05,019 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51898, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T20:14:05,019 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2] 2024-11-18T20:14:05,019 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51926, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T20:14:05,019 DEBUG [Thread-2066 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T20:14:05,019 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51924, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T20:14:05,019 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=121, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-18T20:14:05,019 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=122, ppid=121, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-18T20:14:05,020 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2] 2024-11-18T20:14:05,020 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2] 2024-11-18T20:14:05,020 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2] 2024-11-18T20:14:05,021 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2] 2024-11-18T20:14:05,021 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c0a89b2656d4,35963,1731960718223, seqNum=-1] 2024-11-18T20:14:05,021 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T20:14:05,022 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2] 2024-11-18T20:14:05,023 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51942, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T20:14:05,025 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2] 2024-11-18T20:14:05,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(8855): Flush requested on 23d90021d1eda01703e7bdf7270f74fc 2024-11-18T20:14:05,027 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-18T20:14:05,027 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 23d90021d1eda01703e7bdf7270f74fc 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-18T20:14:05,047 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/.tmp/A/6abb4bda81374263ae4e64a9784e7315 is 50, key is test_row_0/A:col10/1731960845023/Put/seqid=0 2024-11-18T20:14:05,050 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742284_1460 (size=16681) 2024-11-18T20:14:05,103 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:05,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51924 deadline: 1731960905095, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:05,104 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:05,104 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:05,104 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:05,104 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:14:05,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51862 deadline: 1731960905096, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:05,104 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:05,104 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:05,104 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:05,104 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:14:05,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51860 deadline: 1731960905098, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:05,105 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:05,105 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:05,105 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:14:05,105 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:05,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51894 deadline: 1731960905099, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:05,105 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:05,105 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:05,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51878 deadline: 1731960905100, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:05,105 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:05,105 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:14:05,105 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:05,105 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:05,105 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:14:05,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-18T20:14:05,174 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35963 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-18T20:14:05,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc. 2024-11-18T20:14:05,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc. as already flushing 2024-11-18T20:14:05,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc. 2024-11-18T20:14:05,174 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 23d90021d1eda01703e7bdf7270f74fc, NAME => 'TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:14:05,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 23d90021d1eda01703e7bdf7270f74fc, NAME => 'TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:14:05,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 23d90021d1eda01703e7bdf7270f74fc, NAME => 'TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 23d90021d1eda01703e7bdf7270f74fc, NAME => 'TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:14:05,217 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:05,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51924 deadline: 1731960905213, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:05,218 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:05,218 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:05,218 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:05,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51862 deadline: 1731960905213, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:05,218 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:14:05,218 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:05,218 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:05,218 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:14:05,218 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:05,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51860 deadline: 1731960905213, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:05,218 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:05,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51894 deadline: 1731960905213, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:05,218 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:05,218 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:05,218 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:14:05,218 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:05,219 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:05,219 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:05,219 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:14:05,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51878 deadline: 1731960905213, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:05,219 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:05,219 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:05,219 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:14:05,327 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35963 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-18T20:14:05,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc. 2024-11-18T20:14:05,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc. as already flushing 2024-11-18T20:14:05,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc. 2024-11-18T20:14:05,327 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 23d90021d1eda01703e7bdf7270f74fc, NAME => 'TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:14:05,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 23d90021d1eda01703e7bdf7270f74fc, NAME => 'TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:14:05,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 23d90021d1eda01703e7bdf7270f74fc, NAME => 'TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 23d90021d1eda01703e7bdf7270f74fc, NAME => 'TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:14:05,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-18T20:14:05,427 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:05,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51924 deadline: 1731960905423, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:05,427 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:05,427 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:05,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51862 deadline: 1731960905423, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:05,427 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:05,427 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:14:05,427 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:05,427 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:05,427 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:05,427 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:14:05,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51860 deadline: 1731960905423, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:05,428 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:05,428 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:05,428 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:05,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51894 deadline: 1731960905423, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:05,428 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:14:05,428 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:05,428 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:05,428 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:05,428 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:14:05,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51878 deadline: 1731960905424, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:05,428 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:05,428 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:05,428 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:14:05,450 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/.tmp/A/6abb4bda81374263ae4e64a9784e7315 2024-11-18T20:14:05,473 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/.tmp/B/6197655010b24599a7efc3bc3f3d05d7 is 50, key is test_row_0/B:col10/1731960845023/Put/seqid=0 2024-11-18T20:14:05,477 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742285_1461 (size=12001) 2024-11-18T20:14:05,480 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35963 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-18T20:14:05,480 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc. 2024-11-18T20:14:05,480 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc. as already flushing 2024-11-18T20:14:05,480 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc. 2024-11-18T20:14:05,480 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 23d90021d1eda01703e7bdf7270f74fc, NAME => 'TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:14:05,480 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 23d90021d1eda01703e7bdf7270f74fc, NAME => 'TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:14:05,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 23d90021d1eda01703e7bdf7270f74fc, NAME => 'TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 23d90021d1eda01703e7bdf7270f74fc, NAME => 'TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:14:05,632 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35963 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-18T20:14:05,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc. 2024-11-18T20:14:05,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc. as already flushing 2024-11-18T20:14:05,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc. 2024-11-18T20:14:05,633 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 23d90021d1eda01703e7bdf7270f74fc, NAME => 'TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:14:05,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 23d90021d1eda01703e7bdf7270f74fc, NAME => 'TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:14:05,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 23d90021d1eda01703e7bdf7270f74fc, NAME => 'TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 23d90021d1eda01703e7bdf7270f74fc, NAME => 'TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:14:05,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-18T20:14:05,737 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:05,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51924 deadline: 1731960905733, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:05,737 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:05,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51862 deadline: 1731960905733, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:05,737 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:05,737 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:05,737 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:14:05,737 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:05,737 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:05,737 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:05,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51860 deadline: 1731960905733, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:05,737 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:14:05,738 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:05,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51894 deadline: 1731960905733, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:05,738 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:05,738 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:05,738 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:14:05,738 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:05,738 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:05,738 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:05,738 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:14:05,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51878 deadline: 1731960905733, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:05,738 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:05,738 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:05,738 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:14:05,785 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35963 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-18T20:14:05,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc. 2024-11-18T20:14:05,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc. as already flushing 2024-11-18T20:14:05,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc. 2024-11-18T20:14:05,785 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 23d90021d1eda01703e7bdf7270f74fc, NAME => 'TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:14:05,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 23d90021d1eda01703e7bdf7270f74fc, NAME => 'TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:14:05,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 23d90021d1eda01703e7bdf7270f74fc, NAME => 'TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 23d90021d1eda01703e7bdf7270f74fc, NAME => 'TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:14:05,877 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/.tmp/B/6197655010b24599a7efc3bc3f3d05d7 2024-11-18T20:14:05,901 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/.tmp/C/658015f09d4547fe95f8b60bdb49d975 is 50, key is test_row_0/C:col10/1731960845023/Put/seqid=0 2024-11-18T20:14:05,905 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742286_1462 (size=12001) 2024-11-18T20:14:05,905 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/.tmp/C/658015f09d4547fe95f8b60bdb49d975 2024-11-18T20:14:05,908 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/.tmp/A/6abb4bda81374263ae4e64a9784e7315 as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/A/6abb4bda81374263ae4e64a9784e7315 2024-11-18T20:14:05,912 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/A/6abb4bda81374263ae4e64a9784e7315, entries=250, sequenceid=13, filesize=16.3 K 2024-11-18T20:14:05,913 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/.tmp/B/6197655010b24599a7efc3bc3f3d05d7 as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/B/6197655010b24599a7efc3bc3f3d05d7 2024-11-18T20:14:05,915 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/B/6197655010b24599a7efc3bc3f3d05d7, entries=150, sequenceid=13, filesize=11.7 K 2024-11-18T20:14:05,916 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/.tmp/C/658015f09d4547fe95f8b60bdb49d975 as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/C/658015f09d4547fe95f8b60bdb49d975 2024-11-18T20:14:05,918 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/C/658015f09d4547fe95f8b60bdb49d975, entries=150, sequenceid=13, filesize=11.7 K 2024-11-18T20:14:05,919 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 23d90021d1eda01703e7bdf7270f74fc in 892ms, sequenceid=13, compaction requested=false 2024-11-18T20:14:05,920 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 23d90021d1eda01703e7bdf7270f74fc: 2024-11-18T20:14:05,938 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35963 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-18T20:14:05,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc. 2024-11-18T20:14:05,938 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2902): Flushing 23d90021d1eda01703e7bdf7270f74fc 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-18T20:14:05,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/.tmp/A/2b481fd570ec4617883560c22a3cd508 is 50, key is test_row_0/A:col10/1731960845097/Put/seqid=0 2024-11-18T20:14:05,948 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742287_1463 (size=12001) 2024-11-18T20:14:06,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-18T20:14:06,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(8855): Flush requested on 23d90021d1eda01703e7bdf7270f74fc 2024-11-18T20:14:06,247 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc. as already flushing 2024-11-18T20:14:06,260 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:06,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51924 deadline: 1731960906251, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:06,260 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:06,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51862 deadline: 1731960906258, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:06,260 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:06,260 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:06,260 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:14:06,260 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:06,260 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:06,260 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:14:06,266 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:06,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51860 deadline: 1731960906259, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:06,266 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:06,267 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:06,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51894 deadline: 1731960906260, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:06,267 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:06,267 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:14:06,267 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:06,267 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:06,267 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:06,267 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:14:06,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51878 deadline: 1731960906260, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:06,267 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:06,267 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:06,267 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:14:06,348 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/.tmp/A/2b481fd570ec4617883560c22a3cd508 2024-11-18T20:14:06,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/.tmp/B/3f2da01bd90a44998ed58b31aea32a0f is 50, key is test_row_0/B:col10/1731960845097/Put/seqid=0 2024-11-18T20:14:06,357 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742288_1464 (size=12001) 2024-11-18T20:14:06,367 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:06,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51924 deadline: 1731960906363, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:06,368 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:06,368 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:06,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51862 deadline: 1731960906363, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:06,368 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:06,368 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:14:06,368 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:06,368 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:06,368 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:14:06,375 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:06,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51860 deadline: 1731960906373, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:06,375 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:06,375 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:06,375 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:06,375 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:14:06,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51894 deadline: 1731960906373, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:06,376 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:06,376 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:06,376 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:06,376 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:14:06,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51878 deadline: 1731960906373, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:06,376 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:06,376 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:06,376 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:14:06,576 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:06,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51924 deadline: 1731960906572, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:06,577 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:06,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51862 deadline: 1731960906572, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:06,577 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:06,577 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:06,577 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:14:06,577 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:06,577 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:06,577 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:14:06,586 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:06,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51860 deadline: 1731960906583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:06,587 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:06,587 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:06,587 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:06,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51894 deadline: 1731960906583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:06,587 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:14:06,587 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:06,587 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:06,587 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:06,587 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:14:06,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51878 deadline: 1731960906583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:06,587 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:06,587 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:06,587 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:14:06,758 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/.tmp/B/3f2da01bd90a44998ed58b31aea32a0f 2024-11-18T20:14:06,764 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/.tmp/C/dee21cd8e9ca4b3e9b48b4a500324af2 is 50, key is test_row_0/C:col10/1731960845097/Put/seqid=0 2024-11-18T20:14:06,767 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742289_1465 (size=12001) 2024-11-18T20:14:06,889 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:06,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51862 deadline: 1731960906883, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:06,889 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:06,889 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:06,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51924 deadline: 1731960906883, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:06,889 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:06,889 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:14:06,889 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:06,889 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:06,889 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:14:06,896 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:06,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51860 deadline: 1731960906893, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:06,897 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:06,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51878 deadline: 1731960906893, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:06,897 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:06,897 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:06,897 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:14:06,897 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:06,897 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:06,897 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:06,897 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:14:06,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51894 deadline: 1731960906893, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:06,897 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:06,898 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:06,898 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:14:07,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-18T20:14:07,168 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/.tmp/C/dee21cd8e9ca4b3e9b48b4a500324af2 2024-11-18T20:14:07,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/.tmp/A/2b481fd570ec4617883560c22a3cd508 as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/A/2b481fd570ec4617883560c22a3cd508 2024-11-18T20:14:07,174 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/A/2b481fd570ec4617883560c22a3cd508, entries=150, sequenceid=37, filesize=11.7 K 2024-11-18T20:14:07,175 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/.tmp/B/3f2da01bd90a44998ed58b31aea32a0f as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/B/3f2da01bd90a44998ed58b31aea32a0f 2024-11-18T20:14:07,178 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/B/3f2da01bd90a44998ed58b31aea32a0f, entries=150, sequenceid=37, filesize=11.7 K 2024-11-18T20:14:07,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/.tmp/C/dee21cd8e9ca4b3e9b48b4a500324af2 as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/C/dee21cd8e9ca4b3e9b48b4a500324af2 2024-11-18T20:14:07,182 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/C/dee21cd8e9ca4b3e9b48b4a500324af2, entries=150, sequenceid=37, filesize=11.7 K 2024-11-18T20:14:07,183 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=67.09 KB/68700 for 23d90021d1eda01703e7bdf7270f74fc in 1245ms, sequenceid=37, compaction requested=false 2024-11-18T20:14:07,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2603): Flush status journal for 23d90021d1eda01703e7bdf7270f74fc: 2024-11-18T20:14:07,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc. 2024-11-18T20:14:07,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=122 2024-11-18T20:14:07,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.HMaster(4169): Remote procedure done, pid=122 2024-11-18T20:14:07,186 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=122, resume processing ppid=121 2024-11-18T20:14:07,186 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=122, ppid=121, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1650 sec 2024-11-18T20:14:07,187 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=121, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees in 2.1720 sec 2024-11-18T20:14:07,389 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-18T20:14:07,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(8855): Flush requested on 23d90021d1eda01703e7bdf7270f74fc 2024-11-18T20:14:07,406 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-18T20:14:07,407 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 23d90021d1eda01703e7bdf7270f74fc 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-18T20:14:07,411 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/.tmp/A/a4912fa3a4e641bba79c237d7b0e5d29 is 50, key is test_row_0/A:col10/1731960847405/Put/seqid=0 2024-11-18T20:14:07,422 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742290_1466 (size=14341) 2024-11-18T20:14:07,451 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:07,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51878 deadline: 1731960907439, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:07,451 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:07,451 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:07,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51894 deadline: 1731960907440, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:07,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51860 deadline: 1731960907440, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:07,451 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:07,452 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:07,452 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:14:07,452 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:07,452 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:07,452 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:07,452 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:07,452 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:07,452 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:14:07,452 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:14:07,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51862 deadline: 1731960907441, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:07,452 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:07,452 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:07,452 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:07,452 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:14:07,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51924 deadline: 1731960907443, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:07,452 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:07,452 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:07,452 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:14:07,569 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:07,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51878 deadline: 1731960907563, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:07,570 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:07,570 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:07,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51894 deadline: 1731960907563, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:07,570 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:07,570 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:14:07,570 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:07,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51860 deadline: 1731960907563, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:07,570 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:07,570 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:07,570 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:14:07,570 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:07,570 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:07,570 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:07,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51862 deadline: 1731960907563, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:07,570 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:14:07,570 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:07,570 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:07,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51924 deadline: 1731960907563, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:07,570 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:07,570 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:14:07,571 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:07,571 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:07,571 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:14:07,779 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:07,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51878 deadline: 1731960907773, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:07,779 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:07,779 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:07,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51894 deadline: 1731960907773, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:07,779 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:07,779 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:14:07,780 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:07,780 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:07,780 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:07,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51860 deadline: 1731960907773, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:07,780 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:14:07,780 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:07,780 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:07,780 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:07,780 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:14:07,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51862 deadline: 1731960907773, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:07,780 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:07,780 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:07,780 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:14:07,789 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:07,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51924 deadline: 1731960907783, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:07,789 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:07,789 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:07,789 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:14:07,823 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/.tmp/A/a4912fa3a4e641bba79c237d7b0e5d29 2024-11-18T20:14:07,829 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/.tmp/B/320422ad4af1402c8bd6884965309481 is 50, key is test_row_0/B:col10/1731960847405/Put/seqid=0 2024-11-18T20:14:07,833 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742291_1467 (size=12001) 2024-11-18T20:14:07,833 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/.tmp/B/320422ad4af1402c8bd6884965309481 2024-11-18T20:14:07,840 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/.tmp/C/73dd2270193d4a0eb5977663ad0c9c57 is 50, key is test_row_0/C:col10/1731960847405/Put/seqid=0 2024-11-18T20:14:07,843 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742292_1468 (size=12001) 2024-11-18T20:14:08,087 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:08,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51878 deadline: 1731960908083, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:08,087 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:08,087 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:08,087 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:14:08,089 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:08,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51894 deadline: 1731960908083, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:08,090 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:08,090 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:08,090 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:14:08,099 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:08,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51860 deadline: 1731960908093, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:08,099 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:08,099 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:08,099 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:08,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51862 deadline: 1731960908093, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:08,099 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:14:08,100 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:08,100 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:08,100 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:08,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51924 deadline: 1731960908093, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:08,100 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:14:08,100 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:08,100 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:08,100 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:14:08,244 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/.tmp/C/73dd2270193d4a0eb5977663ad0c9c57 2024-11-18T20:14:08,247 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/.tmp/A/a4912fa3a4e641bba79c237d7b0e5d29 as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/A/a4912fa3a4e641bba79c237d7b0e5d29 2024-11-18T20:14:08,250 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/A/a4912fa3a4e641bba79c237d7b0e5d29, entries=200, sequenceid=51, filesize=14.0 K 2024-11-18T20:14:08,250 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/.tmp/B/320422ad4af1402c8bd6884965309481 as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/B/320422ad4af1402c8bd6884965309481 2024-11-18T20:14:08,253 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/B/320422ad4af1402c8bd6884965309481, entries=150, sequenceid=51, filesize=11.7 K 2024-11-18T20:14:08,253 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/.tmp/C/73dd2270193d4a0eb5977663ad0c9c57 as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/C/73dd2270193d4a0eb5977663ad0c9c57 2024-11-18T20:14:08,257 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/C/73dd2270193d4a0eb5977663ad0c9c57, entries=150, sequenceid=51, filesize=11.7 K 2024-11-18T20:14:08,257 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for 23d90021d1eda01703e7bdf7270f74fc in 851ms, sequenceid=51, compaction requested=true 2024-11-18T20:14:08,257 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 23d90021d1eda01703e7bdf7270f74fc: 2024-11-18T20:14:08,258 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T20:14:08,258 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 23d90021d1eda01703e7bdf7270f74fc:A, priority=-2147483648, current under compaction store size is 1 2024-11-18T20:14:08,258 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:14:08,258 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 23d90021d1eda01703e7bdf7270f74fc:B, priority=-2147483648, current under compaction store size is 2 2024-11-18T20:14:08,258 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:14:08,258 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T20:14:08,258 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 23d90021d1eda01703e7bdf7270f74fc:C, priority=-2147483648, current under compaction store size is 3 2024-11-18T20:14:08,258 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T20:14:08,259 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 43023 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T20:14:08,259 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HStore(1541): 23d90021d1eda01703e7bdf7270f74fc/A is initiating minor compaction (all files) 2024-11-18T20:14:08,259 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 23d90021d1eda01703e7bdf7270f74fc/A in TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc. 2024-11-18T20:14:08,259 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T20:14:08,259 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HStore(1541): 23d90021d1eda01703e7bdf7270f74fc/B is initiating minor compaction (all files) 2024-11-18T20:14:08,259 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/A/6abb4bda81374263ae4e64a9784e7315, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/A/2b481fd570ec4617883560c22a3cd508, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/A/a4912fa3a4e641bba79c237d7b0e5d29] into tmpdir=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/.tmp, totalSize=42.0 K 2024-11-18T20:14:08,259 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 23d90021d1eda01703e7bdf7270f74fc/B in TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc. 2024-11-18T20:14:08,259 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/B/6197655010b24599a7efc3bc3f3d05d7, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/B/3f2da01bd90a44998ed58b31aea32a0f, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/B/320422ad4af1402c8bd6884965309481] into tmpdir=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/.tmp, totalSize=35.2 K 2024-11-18T20:14:08,259 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6abb4bda81374263ae4e64a9784e7315, keycount=250, bloomtype=ROW, size=16.3 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1731960845022 2024-11-18T20:14:08,259 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.Compactor(225): Compacting 6197655010b24599a7efc3bc3f3d05d7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1731960845023 2024-11-18T20:14:08,259 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2b481fd570ec4617883560c22a3cd508, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1731960845097 2024-11-18T20:14:08,259 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.Compactor(225): Compacting 3f2da01bd90a44998ed58b31aea32a0f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1731960845097 2024-11-18T20:14:08,260 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] compactions.Compactor(225): Compacting a4912fa3a4e641bba79c237d7b0e5d29, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1731960846247 2024-11-18T20:14:08,260 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.Compactor(225): Compacting 320422ad4af1402c8bd6884965309481, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1731960846247 2024-11-18T20:14:08,266 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 23d90021d1eda01703e7bdf7270f74fc#B#compaction#392 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:14:08,266 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/.tmp/B/7f701157eb3848bf80bb4c47bb2437bc is 50, key is test_row_0/B:col10/1731960847405/Put/seqid=0 2024-11-18T20:14:08,275 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 23d90021d1eda01703e7bdf7270f74fc#A#compaction#393 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:14:08,275 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/.tmp/A/485401aaa2e542e9867ebcd5cbbc3a65 is 50, key is test_row_0/A:col10/1731960847405/Put/seqid=0 2024-11-18T20:14:08,278 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742293_1469 (size=12104) 2024-11-18T20:14:08,285 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742294_1470 (size=12104) 2024-11-18T20:14:08,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(8855): Flush requested on 23d90021d1eda01703e7bdf7270f74fc 2024-11-18T20:14:08,597 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-18T20:14:08,598 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 23d90021d1eda01703e7bdf7270f74fc 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-18T20:14:08,601 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/.tmp/A/ed677c7cbfda479db4cdf6c576e2c22a is 50, key is test_row_0/A:col10/1731960848596/Put/seqid=0 2024-11-18T20:14:08,605 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742295_1471 (size=14341) 2024-11-18T20:14:08,623 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:08,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51894 deadline: 1731960908614, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:08,623 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:08,623 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:08,623 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:08,623 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:14:08,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51924 deadline: 1731960908620, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:08,623 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:08,623 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:08,623 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:14:08,629 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:08,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51860 deadline: 1731960908621, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:08,629 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:08,629 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:08,629 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:08,629 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:14:08,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51878 deadline: 1731960908621, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:08,630 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:08,630 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:08,630 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:14:08,634 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:08,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51862 deadline: 1731960908622, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:08,635 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:08,635 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:08,635 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:14:08,683 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/.tmp/B/7f701157eb3848bf80bb4c47bb2437bc as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/B/7f701157eb3848bf80bb4c47bb2437bc 2024-11-18T20:14:08,688 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 23d90021d1eda01703e7bdf7270f74fc/B of 23d90021d1eda01703e7bdf7270f74fc into 7f701157eb3848bf80bb4c47bb2437bc(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:14:08,688 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 23d90021d1eda01703e7bdf7270f74fc: 2024-11-18T20:14:08,688 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., storeName=23d90021d1eda01703e7bdf7270f74fc/B, priority=13, startTime=1731960848258; duration=0sec 2024-11-18T20:14:08,688 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T20:14:08,688 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 23d90021d1eda01703e7bdf7270f74fc:B 2024-11-18T20:14:08,688 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T20:14:08,689 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T20:14:08,689 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/.tmp/A/485401aaa2e542e9867ebcd5cbbc3a65 as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/A/485401aaa2e542e9867ebcd5cbbc3a65 2024-11-18T20:14:08,689 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HStore(1541): 23d90021d1eda01703e7bdf7270f74fc/C is initiating minor compaction (all files) 2024-11-18T20:14:08,689 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 23d90021d1eda01703e7bdf7270f74fc/C in TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc. 2024-11-18T20:14:08,689 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/C/658015f09d4547fe95f8b60bdb49d975, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/C/dee21cd8e9ca4b3e9b48b4a500324af2, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/C/73dd2270193d4a0eb5977663ad0c9c57] into tmpdir=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/.tmp, totalSize=35.2 K 2024-11-18T20:14:08,689 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.Compactor(225): Compacting 658015f09d4547fe95f8b60bdb49d975, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1731960845023 2024-11-18T20:14:08,689 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.Compactor(225): Compacting dee21cd8e9ca4b3e9b48b4a500324af2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1731960845097 2024-11-18T20:14:08,690 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.Compactor(225): Compacting 73dd2270193d4a0eb5977663ad0c9c57, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1731960846247 2024-11-18T20:14:08,693 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 23d90021d1eda01703e7bdf7270f74fc/A of 23d90021d1eda01703e7bdf7270f74fc into 485401aaa2e542e9867ebcd5cbbc3a65(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:14:08,693 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 23d90021d1eda01703e7bdf7270f74fc: 2024-11-18T20:14:08,693 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., storeName=23d90021d1eda01703e7bdf7270f74fc/A, priority=13, startTime=1731960848257; duration=0sec 2024-11-18T20:14:08,693 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:14:08,693 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 23d90021d1eda01703e7bdf7270f74fc:A 2024-11-18T20:14:08,695 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 23d90021d1eda01703e7bdf7270f74fc#C#compaction#395 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:14:08,696 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/.tmp/C/a67f78075b544420a031fe3472f73b5f is 50, key is test_row_0/C:col10/1731960847405/Put/seqid=0 2024-11-18T20:14:08,699 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742296_1472 (size=12104) 2024-11-18T20:14:08,736 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:08,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51894 deadline: 1731960908732, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:08,736 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:08,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51924 deadline: 1731960908733, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:08,737 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:08,737 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:08,737 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:14:08,737 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:08,737 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:08,737 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:08,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51860 deadline: 1731960908733, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:08,737 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:14:08,737 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:08,737 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:08,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51878 deadline: 1731960908733, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:08,738 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:08,738 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:14:08,738 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:08,738 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:08,738 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:14:08,745 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:08,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51862 deadline: 1731960908743, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:08,745 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:08,745 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:08,746 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:14:08,946 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:08,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51894 deadline: 1731960908943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:08,947 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:08,947 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:08,947 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:14:08,949 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:08,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51924 deadline: 1731960908943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:08,949 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:08,949 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:08,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51878 deadline: 1731960908943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:08,949 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:08,949 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:14:08,950 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:08,950 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:08,950 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:08,950 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:14:08,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51860 deadline: 1731960908943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:08,950 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:08,950 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:08,950 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:14:08,957 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:08,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51862 deadline: 1731960908953, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:08,958 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:08,958 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:08,958 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:14:09,005 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=74 (bloomFilter=true), to=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/.tmp/A/ed677c7cbfda479db4cdf6c576e2c22a 2024-11-18T20:14:09,011 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/.tmp/B/1be07d98d42a41d68f4ac89fc7a9513a is 50, key is test_row_0/B:col10/1731960848596/Put/seqid=0 2024-11-18T20:14:09,015 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742297_1473 (size=12001) 2024-11-18T20:14:09,102 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/.tmp/C/a67f78075b544420a031fe3472f73b5f as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/C/a67f78075b544420a031fe3472f73b5f 2024-11-18T20:14:09,106 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 23d90021d1eda01703e7bdf7270f74fc/C of 23d90021d1eda01703e7bdf7270f74fc into a67f78075b544420a031fe3472f73b5f(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:14:09,106 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 23d90021d1eda01703e7bdf7270f74fc: 2024-11-18T20:14:09,106 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., storeName=23d90021d1eda01703e7bdf7270f74fc/C, priority=13, startTime=1731960848258; duration=0sec 2024-11-18T20:14:09,106 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:14:09,106 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 23d90021d1eda01703e7bdf7270f74fc:C 2024-11-18T20:14:09,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-18T20:14:09,173 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-18T20:14:09,173 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-18T20:14:09,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] procedure2.ProcedureExecutor(1139): Stored pid=123, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees 2024-11-18T20:14:09,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-18T20:14:09,175 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=123, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-18T20:14:09,176 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=123, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-18T20:14:09,176 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=124, ppid=123, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-18T20:14:09,255 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:09,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51894 deadline: 1731960909253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:09,255 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:09,256 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:09,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51924 deadline: 1731960909253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:09,256 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:09,256 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:14:09,256 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:09,256 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:09,256 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:09,256 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:14:09,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51878 deadline: 1731960909253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:09,256 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:09,256 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:09,256 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:09,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51860 deadline: 1731960909253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:09,256 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:14:09,256 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:09,256 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:09,256 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:14:09,267 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:09,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51862 deadline: 1731960909263, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:09,268 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:09,268 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:09,268 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:14:09,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-18T20:14:09,328 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35963 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-18T20:14:09,328 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc. 2024-11-18T20:14:09,328 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc. as already flushing 2024-11-18T20:14:09,328 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc. 2024-11-18T20:14:09,328 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 23d90021d1eda01703e7bdf7270f74fc, NAME => 'TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:14:09,328 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 23d90021d1eda01703e7bdf7270f74fc, NAME => 'TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:14:09,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 23d90021d1eda01703e7bdf7270f74fc, NAME => 'TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 23d90021d1eda01703e7bdf7270f74fc, NAME => 'TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:14:09,416 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=74 (bloomFilter=true), to=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/.tmp/B/1be07d98d42a41d68f4ac89fc7a9513a 2024-11-18T20:14:09,421 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/.tmp/C/d8ae2617f2e24a0c9f7853d4f71f70a3 is 50, key is test_row_0/C:col10/1731960848596/Put/seqid=0 2024-11-18T20:14:09,425 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742298_1474 (size=12001) 2024-11-18T20:14:09,481 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35963 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-18T20:14:09,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc. 2024-11-18T20:14:09,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc. as already flushing 2024-11-18T20:14:09,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc. 2024-11-18T20:14:09,481 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 23d90021d1eda01703e7bdf7270f74fc, NAME => 'TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:14:09,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 23d90021d1eda01703e7bdf7270f74fc, NAME => 'TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:14:09,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 23d90021d1eda01703e7bdf7270f74fc, NAME => 'TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 23d90021d1eda01703e7bdf7270f74fc, NAME => 'TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:14:09,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-18T20:14:09,633 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35963 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-18T20:14:09,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc. 2024-11-18T20:14:09,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc. as already flushing 2024-11-18T20:14:09,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc. 2024-11-18T20:14:09,633 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 23d90021d1eda01703e7bdf7270f74fc, NAME => 'TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:14:09,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 23d90021d1eda01703e7bdf7270f74fc, NAME => 'TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:14:09,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 23d90021d1eda01703e7bdf7270f74fc, NAME => 'TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 23d90021d1eda01703e7bdf7270f74fc, NAME => 'TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:14:09,765 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:09,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51894 deadline: 1731960909764, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:09,765 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:09,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51924 deadline: 1731960909764, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:09,766 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:09,766 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:09,766 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:14:09,766 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:09,766 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:09,766 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:09,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51878 deadline: 1731960909764, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:09,766 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:14:09,766 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:09,766 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:09,766 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:09,766 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:14:09,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51860 deadline: 1731960909764, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:09,766 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:09,766 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:09,766 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:14:09,778 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:09,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51862 deadline: 1731960909774, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:09,779 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:09,779 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:09,779 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:14:09,786 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35963 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-18T20:14:09,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc. 2024-11-18T20:14:09,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc. as already flushing 2024-11-18T20:14:09,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc. 2024-11-18T20:14:09,786 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 23d90021d1eda01703e7bdf7270f74fc, NAME => 'TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:14:09,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 23d90021d1eda01703e7bdf7270f74fc, NAME => 'TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:14:09,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 23d90021d1eda01703e7bdf7270f74fc, NAME => 'TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 23d90021d1eda01703e7bdf7270f74fc, NAME => 'TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:14:09,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-18T20:14:09,825 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=74 (bloomFilter=true), to=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/.tmp/C/d8ae2617f2e24a0c9f7853d4f71f70a3 2024-11-18T20:14:09,830 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/.tmp/A/ed677c7cbfda479db4cdf6c576e2c22a as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/A/ed677c7cbfda479db4cdf6c576e2c22a 2024-11-18T20:14:09,842 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/A/ed677c7cbfda479db4cdf6c576e2c22a, entries=200, sequenceid=74, filesize=14.0 K 2024-11-18T20:14:09,843 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/.tmp/B/1be07d98d42a41d68f4ac89fc7a9513a as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/B/1be07d98d42a41d68f4ac89fc7a9513a 2024-11-18T20:14:09,846 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/B/1be07d98d42a41d68f4ac89fc7a9513a, entries=150, sequenceid=74, filesize=11.7 K 2024-11-18T20:14:09,847 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/.tmp/C/d8ae2617f2e24a0c9f7853d4f71f70a3 as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/C/d8ae2617f2e24a0c9f7853d4f71f70a3 2024-11-18T20:14:09,853 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/C/d8ae2617f2e24a0c9f7853d4f71f70a3, entries=150, sequenceid=74, filesize=11.7 K 2024-11-18T20:14:09,853 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for 23d90021d1eda01703e7bdf7270f74fc in 1256ms, sequenceid=74, compaction requested=false 2024-11-18T20:14:09,853 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 23d90021d1eda01703e7bdf7270f74fc: 2024-11-18T20:14:09,939 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35963 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-18T20:14:09,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc. 2024-11-18T20:14:09,939 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2902): Flushing 23d90021d1eda01703e7bdf7270f74fc 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-18T20:14:09,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/.tmp/A/dbbe3593039b40ecb785aabbefcfa7b1 is 50, key is test_row_0/A:col10/1731960848621/Put/seqid=0 2024-11-18T20:14:09,949 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742299_1475 (size=12001) 2024-11-18T20:14:10,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-18T20:14:10,349 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/.tmp/A/dbbe3593039b40ecb785aabbefcfa7b1 2024-11-18T20:14:10,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/.tmp/B/5d0d101881774c259e9a9497939c0fdf is 50, key is test_row_0/B:col10/1731960848621/Put/seqid=0 2024-11-18T20:14:10,362 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742300_1476 (size=12001) 2024-11-18T20:14:10,763 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/.tmp/B/5d0d101881774c259e9a9497939c0fdf 2024-11-18T20:14:10,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/.tmp/C/58f7a3be119d4201af7f9b9c8c0fe99d is 50, key is test_row_0/C:col10/1731960848621/Put/seqid=0 2024-11-18T20:14:10,771 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742301_1477 (size=12001) 2024-11-18T20:14:10,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(8855): Flush requested on 23d90021d1eda01703e7bdf7270f74fc 2024-11-18T20:14:10,775 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc. as already flushing 2024-11-18T20:14:10,813 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:10,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51860 deadline: 1731960910803, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:10,814 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:10,814 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:10,814 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:10,814 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:14:10,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51894 deadline: 1731960910808, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:10,814 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:10,814 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:10,814 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:14:10,814 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:10,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51862 deadline: 1731960910809, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:10,814 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:10,814 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:10,814 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:14:10,817 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:10,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51878 deadline: 1731960910813, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:10,818 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:10,818 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:10,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51924 deadline: 1731960910813, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:10,818 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:10,818 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:14:10,818 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:10,818 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:10,818 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:14:10,925 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:10,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51860 deadline: 1731960910923, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:10,925 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:10,925 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:10,925 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:10,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51894 deadline: 1731960910923, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:10,925 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:14:10,925 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:10,925 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:10,926 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:10,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51862 deadline: 1731960910923, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:10,926 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:14:10,926 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:10,926 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:10,926 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:14:10,930 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:10,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51924 deadline: 1731960910923, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:10,930 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:10,930 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:10,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51878 deadline: 1731960910923, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:10,930 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:10,930 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:14:10,930 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:10,930 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:10,930 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:14:11,140 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:11,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51860 deadline: 1731960911133, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:11,140 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:11,140 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:11,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51894 deadline: 1731960911133, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:11,140 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:11,140 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:14:11,140 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:11,140 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:11,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51862 deadline: 1731960911133, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:11,140 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:11,140 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:14:11,140 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:11,141 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:11,141 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:11,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51924 deadline: 1731960911133, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:11,141 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:14:11,141 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:11,141 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:11,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51878 deadline: 1731960911133, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:11,141 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:11,141 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:14:11,141 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:11,141 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:11,141 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:14:11,172 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/.tmp/C/58f7a3be119d4201af7f9b9c8c0fe99d 2024-11-18T20:14:11,175 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/.tmp/A/dbbe3593039b40ecb785aabbefcfa7b1 as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/A/dbbe3593039b40ecb785aabbefcfa7b1 2024-11-18T20:14:11,179 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/A/dbbe3593039b40ecb785aabbefcfa7b1, entries=150, sequenceid=90, filesize=11.7 K 2024-11-18T20:14:11,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/.tmp/B/5d0d101881774c259e9a9497939c0fdf as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/B/5d0d101881774c259e9a9497939c0fdf 2024-11-18T20:14:11,182 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/B/5d0d101881774c259e9a9497939c0fdf, entries=150, sequenceid=90, filesize=11.7 K 2024-11-18T20:14:11,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/.tmp/C/58f7a3be119d4201af7f9b9c8c0fe99d as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/C/58f7a3be119d4201af7f9b9c8c0fe99d 2024-11-18T20:14:11,185 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/C/58f7a3be119d4201af7f9b9c8c0fe99d, entries=150, sequenceid=90, filesize=11.7 K 2024-11-18T20:14:11,186 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=140.89 KB/144270 for 23d90021d1eda01703e7bdf7270f74fc in 1247ms, sequenceid=90, compaction requested=true 2024-11-18T20:14:11,186 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2603): Flush status journal for 23d90021d1eda01703e7bdf7270f74fc: 2024-11-18T20:14:11,186 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc. 2024-11-18T20:14:11,186 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=124 2024-11-18T20:14:11,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.HMaster(4169): Remote procedure done, pid=124 2024-11-18T20:14:11,188 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=124, resume processing ppid=123 2024-11-18T20:14:11,188 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=124, ppid=123, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0110 sec 2024-11-18T20:14:11,189 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=123, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees in 2.0150 sec 2024-11-18T20:14:11,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-18T20:14:11,323 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-18T20:14:11,323 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-18T20:14:11,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] procedure2.ProcedureExecutor(1139): Stored pid=125, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees 2024-11-18T20:14:11,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-18T20:14:11,325 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=125, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-18T20:14:11,325 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=125, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-18T20:14:11,326 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=126, ppid=125, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-18T20:14:11,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-18T20:14:11,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(8855): Flush requested on 23d90021d1eda01703e7bdf7270f74fc 2024-11-18T20:14:11,456 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-18T20:14:11,456 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 23d90021d1eda01703e7bdf7270f74fc 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-18T20:14:11,459 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/.tmp/A/f37bd9a4a924422fb52549c4e340e073 is 50, key is test_row_0/A:col10/1731960850800/Put/seqid=0 2024-11-18T20:14:11,464 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742302_1478 (size=14341) 2024-11-18T20:14:11,470 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:11,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51878 deadline: 1731960911460, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:11,471 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:11,471 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:11,471 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:14:11,474 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:11,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51860 deadline: 1731960911463, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:11,474 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:11,474 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:11,474 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:11,474 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:14:11,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51894 deadline: 1731960911469, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:11,475 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:11,475 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:11,475 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:14:11,477 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35963 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-18T20:14:11,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc. 2024-11-18T20:14:11,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc. as already flushing 2024-11-18T20:14:11,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc. 2024-11-18T20:14:11,477 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 23d90021d1eda01703e7bdf7270f74fc, NAME => 'TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:14:11,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 23d90021d1eda01703e7bdf7270f74fc, NAME => 'TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:14:11,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 23d90021d1eda01703e7bdf7270f74fc, NAME => 'TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 23d90021d1eda01703e7bdf7270f74fc, NAME => 'TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:14:11,481 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:11,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51862 deadline: 1731960911470, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:11,481 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:11,481 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:11,481 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:11,481 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:14:11,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51924 deadline: 1731960911471, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:11,482 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:11,482 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:11,482 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:14:11,581 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:11,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51878 deadline: 1731960911573, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:11,581 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:11,581 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:11,581 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:14:11,590 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:11,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51860 deadline: 1731960911582, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:11,590 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:11,590 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:11,590 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:14:11,590 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:11,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51894 deadline: 1731960911583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:11,591 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:11,591 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:11,591 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:14:11,591 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:11,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51862 deadline: 1731960911583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:11,591 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:11,591 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:11,591 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:14:11,600 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:11,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51924 deadline: 1731960911593, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:11,600 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:11,600 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:11,600 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:14:11,630 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35963 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-18T20:14:11,630 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc. 2024-11-18T20:14:11,630 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc. as already flushing 2024-11-18T20:14:11,630 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc. 2024-11-18T20:14:11,630 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 23d90021d1eda01703e7bdf7270f74fc, NAME => 'TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:14:11,630 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 23d90021d1eda01703e7bdf7270f74fc, NAME => 'TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:14:11,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 23d90021d1eda01703e7bdf7270f74fc, NAME => 'TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 23d90021d1eda01703e7bdf7270f74fc, NAME => 'TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:14:11,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-18T20:14:11,782 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35963 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-18T20:14:11,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc. 2024-11-18T20:14:11,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc. as already flushing 2024-11-18T20:14:11,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc. 2024-11-18T20:14:11,783 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 23d90021d1eda01703e7bdf7270f74fc, NAME => 'TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:14:11,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 23d90021d1eda01703e7bdf7270f74fc, NAME => 'TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:14:11,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 23d90021d1eda01703e7bdf7270f74fc, NAME => 'TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 23d90021d1eda01703e7bdf7270f74fc, NAME => 'TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:14:11,798 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:11,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51878 deadline: 1731960911792, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:11,799 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:11,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51894 deadline: 1731960911793, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:11,799 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:11,799 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:11,799 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:14:11,799 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:11,799 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:11,799 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:14:11,809 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:11,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51860 deadline: 1731960911803, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:11,809 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:11,809 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:11,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51862 deadline: 1731960911803, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:11,810 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:11,810 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:14:11,810 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:11,810 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:11,810 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:11,810 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:14:11,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51924 deadline: 1731960911803, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:11,810 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:11,810 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:11,810 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:14:11,864 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=115 (bloomFilter=true), to=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/.tmp/A/f37bd9a4a924422fb52549c4e340e073 2024-11-18T20:14:11,871 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/.tmp/B/99c709c444534070a65020ddb9dcc9ca is 50, key is test_row_0/B:col10/1731960850800/Put/seqid=0 2024-11-18T20:14:11,873 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742303_1479 (size=12001) 2024-11-18T20:14:11,874 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=115 (bloomFilter=true), to=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/.tmp/B/99c709c444534070a65020ddb9dcc9ca 2024-11-18T20:14:11,880 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/.tmp/C/871bd0b0d0cd4d2ea67922e7a41fa62b is 50, key is test_row_0/C:col10/1731960850800/Put/seqid=0 2024-11-18T20:14:11,883 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742304_1480 (size=12001) 2024-11-18T20:14:11,935 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35963 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-18T20:14:11,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc. 2024-11-18T20:14:11,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc. as already flushing 2024-11-18T20:14:11,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc. 2024-11-18T20:14:11,935 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 23d90021d1eda01703e7bdf7270f74fc, NAME => 'TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:14:11,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 23d90021d1eda01703e7bdf7270f74fc, NAME => 'TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:14:11,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 23d90021d1eda01703e7bdf7270f74fc, NAME => 'TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 23d90021d1eda01703e7bdf7270f74fc, NAME => 'TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:14:11,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-18T20:14:12,088 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35963 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-18T20:14:12,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc. 2024-11-18T20:14:12,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc. as already flushing 2024-11-18T20:14:12,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc. 2024-11-18T20:14:12,088 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 23d90021d1eda01703e7bdf7270f74fc, NAME => 'TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:14:12,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 23d90021d1eda01703e7bdf7270f74fc, NAME => 'TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:14:12,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 23d90021d1eda01703e7bdf7270f74fc, NAME => 'TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 23d90021d1eda01703e7bdf7270f74fc, NAME => 'TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:14:12,108 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:12,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51878 deadline: 1731960912103, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:12,109 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:12,109 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:12,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51894 deadline: 1731960912103, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:12,109 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:12,109 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:14:12,109 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:12,109 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:12,109 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:14:12,118 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:12,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51862 deadline: 1731960912113, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:12,118 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:12,118 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:12,118 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:14:12,130 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:12,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51860 deadline: 1731960912123, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:12,130 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:12,130 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:12,130 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:12,130 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:14:12,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51924 deadline: 1731960912123, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:12,130 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:12,130 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:12,130 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:14:12,240 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35963 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-18T20:14:12,240 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc. 2024-11-18T20:14:12,241 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc. as already flushing 2024-11-18T20:14:12,241 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc. 2024-11-18T20:14:12,241 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 23d90021d1eda01703e7bdf7270f74fc, NAME => 'TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:14:12,241 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 23d90021d1eda01703e7bdf7270f74fc, NAME => 'TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:14:12,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 23d90021d1eda01703e7bdf7270f74fc, NAME => 'TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 23d90021d1eda01703e7bdf7270f74fc, NAME => 'TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:14:12,285 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=115 (bloomFilter=true), to=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/.tmp/C/871bd0b0d0cd4d2ea67922e7a41fa62b 2024-11-18T20:14:12,288 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/.tmp/A/f37bd9a4a924422fb52549c4e340e073 as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/A/f37bd9a4a924422fb52549c4e340e073 2024-11-18T20:14:12,290 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/A/f37bd9a4a924422fb52549c4e340e073, entries=200, sequenceid=115, filesize=14.0 K 2024-11-18T20:14:12,291 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/.tmp/B/99c709c444534070a65020ddb9dcc9ca as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/B/99c709c444534070a65020ddb9dcc9ca 2024-11-18T20:14:12,293 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/B/99c709c444534070a65020ddb9dcc9ca, entries=150, sequenceid=115, filesize=11.7 K 2024-11-18T20:14:12,294 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/.tmp/C/871bd0b0d0cd4d2ea67922e7a41fa62b as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/C/871bd0b0d0cd4d2ea67922e7a41fa62b 2024-11-18T20:14:12,297 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/C/871bd0b0d0cd4d2ea67922e7a41fa62b, entries=150, sequenceid=115, filesize=11.7 K 2024-11-18T20:14:12,298 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 23d90021d1eda01703e7bdf7270f74fc in 842ms, sequenceid=115, compaction requested=true 2024-11-18T20:14:12,298 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 23d90021d1eda01703e7bdf7270f74fc: 2024-11-18T20:14:12,298 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 23d90021d1eda01703e7bdf7270f74fc:A, priority=-2147483648, current under compaction store size is 1 2024-11-18T20:14:12,298 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:14:12,298 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-18T20:14:12,298 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 23d90021d1eda01703e7bdf7270f74fc:B, priority=-2147483648, current under compaction store size is 2 2024-11-18T20:14:12,298 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:14:12,298 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-18T20:14:12,298 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 23d90021d1eda01703e7bdf7270f74fc:C, priority=-2147483648, current under compaction store size is 3 2024-11-18T20:14:12,298 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T20:14:12,299 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 52787 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-18T20:14:12,299 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HStore(1541): 23d90021d1eda01703e7bdf7270f74fc/A is initiating minor compaction (all files) 2024-11-18T20:14:12,299 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 23d90021d1eda01703e7bdf7270f74fc/A in TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc. 2024-11-18T20:14:12,299 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/A/485401aaa2e542e9867ebcd5cbbc3a65, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/A/ed677c7cbfda479db4cdf6c576e2c22a, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/A/dbbe3593039b40ecb785aabbefcfa7b1, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/A/f37bd9a4a924422fb52549c4e340e073] into tmpdir=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/.tmp, totalSize=51.5 K 2024-11-18T20:14:12,299 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-18T20:14:12,299 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] compactions.Compactor(225): Compacting 485401aaa2e542e9867ebcd5cbbc3a65, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1731960846247 2024-11-18T20:14:12,299 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HStore(1541): 23d90021d1eda01703e7bdf7270f74fc/B is initiating minor compaction (all files) 2024-11-18T20:14:12,299 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 23d90021d1eda01703e7bdf7270f74fc/B in TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc. 2024-11-18T20:14:12,299 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/B/7f701157eb3848bf80bb4c47bb2437bc, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/B/1be07d98d42a41d68f4ac89fc7a9513a, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/B/5d0d101881774c259e9a9497939c0fdf, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/B/99c709c444534070a65020ddb9dcc9ca] into tmpdir=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/.tmp, totalSize=47.0 K 2024-11-18T20:14:12,299 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] compactions.Compactor(225): Compacting ed677c7cbfda479db4cdf6c576e2c22a, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=74, earliestPutTs=1731960847438 2024-11-18T20:14:12,300 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.Compactor(225): Compacting 7f701157eb3848bf80bb4c47bb2437bc, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1731960846247 2024-11-18T20:14:12,300 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] compactions.Compactor(225): Compacting dbbe3593039b40ecb785aabbefcfa7b1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=90, earliestPutTs=1731960848610 2024-11-18T20:14:12,300 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.Compactor(225): Compacting 1be07d98d42a41d68f4ac89fc7a9513a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=74, earliestPutTs=1731960847438 2024-11-18T20:14:12,300 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] compactions.Compactor(225): Compacting f37bd9a4a924422fb52549c4e340e073, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=115, earliestPutTs=1731960850800 2024-11-18T20:14:12,300 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.Compactor(225): Compacting 5d0d101881774c259e9a9497939c0fdf, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=90, earliestPutTs=1731960848610 2024-11-18T20:14:12,301 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.Compactor(225): Compacting 99c709c444534070a65020ddb9dcc9ca, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=115, earliestPutTs=1731960850800 2024-11-18T20:14:12,307 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 23d90021d1eda01703e7bdf7270f74fc#B#compaction#404 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:14:12,307 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/.tmp/B/d40c2f043c674d86808c3bbc74972a1e is 50, key is test_row_0/B:col10/1731960850800/Put/seqid=0 2024-11-18T20:14:12,310 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742305_1481 (size=12241) 2024-11-18T20:14:12,311 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 23d90021d1eda01703e7bdf7270f74fc#A#compaction#405 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:14:12,312 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/.tmp/A/0de15de0a61b4d81b7cc2edfcfeb51c0 is 50, key is test_row_0/A:col10/1731960850800/Put/seqid=0 2024-11-18T20:14:12,315 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742306_1482 (size=12241) 2024-11-18T20:14:12,393 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35963 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-18T20:14:12,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc. 2024-11-18T20:14:12,394 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2902): Flushing 23d90021d1eda01703e7bdf7270f74fc 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-18T20:14:12,398 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/.tmp/A/f3862c91776c40a48b22b8344ee618e4 is 50, key is test_row_0/A:col10/1731960851469/Put/seqid=0 2024-11-18T20:14:12,405 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742307_1483 (size=12001) 2024-11-18T20:14:12,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-18T20:14:12,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(8855): Flush requested on 23d90021d1eda01703e7bdf7270f74fc 2024-11-18T20:14:12,625 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc. as already flushing 2024-11-18T20:14:12,667 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:12,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51860 deadline: 1731960912659, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:12,667 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:12,667 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:12,667 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:14:12,670 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:12,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51924 deadline: 1731960912660, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:12,670 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:12,670 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:12,670 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:12,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51862 deadline: 1731960912666, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:12,670 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:14:12,671 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:12,671 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:12,671 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:14:12,680 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:12,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51878 deadline: 1731960912666, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:12,680 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:12,680 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:12,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51894 deadline: 1731960912667, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:12,680 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:12,680 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:14:12,680 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:12,680 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:12,680 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:14:12,714 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/.tmp/B/d40c2f043c674d86808c3bbc74972a1e as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/B/d40c2f043c674d86808c3bbc74972a1e 2024-11-18T20:14:12,717 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/.tmp/A/0de15de0a61b4d81b7cc2edfcfeb51c0 as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/A/0de15de0a61b4d81b7cc2edfcfeb51c0 2024-11-18T20:14:12,718 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 23d90021d1eda01703e7bdf7270f74fc/B of 23d90021d1eda01703e7bdf7270f74fc into d40c2f043c674d86808c3bbc74972a1e(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:14:12,718 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 23d90021d1eda01703e7bdf7270f74fc: 2024-11-18T20:14:12,718 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., storeName=23d90021d1eda01703e7bdf7270f74fc/B, priority=12, startTime=1731960852298; duration=0sec 2024-11-18T20:14:12,718 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T20:14:12,718 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 23d90021d1eda01703e7bdf7270f74fc:B 2024-11-18T20:14:12,718 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-18T20:14:12,719 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-18T20:14:12,719 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HStore(1541): 23d90021d1eda01703e7bdf7270f74fc/C is initiating minor compaction (all files) 2024-11-18T20:14:12,719 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 23d90021d1eda01703e7bdf7270f74fc/C in TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc. 2024-11-18T20:14:12,720 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/C/a67f78075b544420a031fe3472f73b5f, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/C/d8ae2617f2e24a0c9f7853d4f71f70a3, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/C/58f7a3be119d4201af7f9b9c8c0fe99d, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/C/871bd0b0d0cd4d2ea67922e7a41fa62b] into tmpdir=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/.tmp, totalSize=47.0 K 2024-11-18T20:14:12,720 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.Compactor(225): Compacting a67f78075b544420a031fe3472f73b5f, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1731960846247 2024-11-18T20:14:12,721 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.Compactor(225): Compacting d8ae2617f2e24a0c9f7853d4f71f70a3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=74, earliestPutTs=1731960847438 2024-11-18T20:14:12,721 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.Compactor(225): Compacting 58f7a3be119d4201af7f9b9c8c0fe99d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=90, earliestPutTs=1731960848610 2024-11-18T20:14:12,721 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.Compactor(225): Compacting 871bd0b0d0cd4d2ea67922e7a41fa62b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=115, earliestPutTs=1731960850800 2024-11-18T20:14:12,721 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 23d90021d1eda01703e7bdf7270f74fc/A of 23d90021d1eda01703e7bdf7270f74fc into 0de15de0a61b4d81b7cc2edfcfeb51c0(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:14:12,722 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 23d90021d1eda01703e7bdf7270f74fc: 2024-11-18T20:14:12,722 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., storeName=23d90021d1eda01703e7bdf7270f74fc/A, priority=12, startTime=1731960852298; duration=0sec 2024-11-18T20:14:12,722 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:14:12,722 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 23d90021d1eda01703e7bdf7270f74fc:A 2024-11-18T20:14:12,727 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 23d90021d1eda01703e7bdf7270f74fc#C#compaction#407 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:14:12,728 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/.tmp/C/ded89bf64a10475ca371570944ef60a9 is 50, key is test_row_0/C:col10/1731960850800/Put/seqid=0 2024-11-18T20:14:12,730 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742308_1484 (size=12241) 2024-11-18T20:14:12,779 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:12,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51860 deadline: 1731960912772, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:12,779 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:12,780 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:12,780 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:14:12,781 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:12,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51924 deadline: 1731960912773, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:12,782 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:12,782 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:12,782 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:12,782 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:14:12,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51862 deadline: 1731960912773, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:12,782 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:12,782 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:12,782 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:14:12,794 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:12,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51878 deadline: 1731960912783, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:12,794 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:12,794 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:12,794 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:12,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51894 deadline: 1731960912783, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:12,794 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:14:12,794 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:12,794 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:12,794 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:14:12,806 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=126 (bloomFilter=true), to=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/.tmp/A/f3862c91776c40a48b22b8344ee618e4 2024-11-18T20:14:12,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/.tmp/B/8c1a458ee4d44b2d867154a3a14ea740 is 50, key is test_row_0/B:col10/1731960851469/Put/seqid=0 2024-11-18T20:14:12,815 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742309_1485 (size=12001) 2024-11-18T20:14:12,985 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:12,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51860 deadline: 1731960912982, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:12,986 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:12,986 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:12,986 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:14:12,995 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:13,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51924 deadline: 1731960912993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:13,001 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:13,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51862 deadline: 1731960912993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:13,002 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:13,002 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:13,002 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:14:13,013 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:13,013 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:13,013 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:14:13,014 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:13,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51878 deadline: 1731960913003, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:13,014 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:13,014 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:13,014 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:13,014 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:14:13,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51894 deadline: 1731960913003, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:13,015 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:13,015 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:13,015 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:14:13,134 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/.tmp/C/ded89bf64a10475ca371570944ef60a9 as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/C/ded89bf64a10475ca371570944ef60a9 2024-11-18T20:14:13,138 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 23d90021d1eda01703e7bdf7270f74fc/C of 23d90021d1eda01703e7bdf7270f74fc into ded89bf64a10475ca371570944ef60a9(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:14:13,138 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 23d90021d1eda01703e7bdf7270f74fc: 2024-11-18T20:14:13,138 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., storeName=23d90021d1eda01703e7bdf7270f74fc/C, priority=12, startTime=1731960852298; duration=0sec 2024-11-18T20:14:13,138 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:14:13,138 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 23d90021d1eda01703e7bdf7270f74fc:C 2024-11-18T20:14:13,216 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=126 (bloomFilter=true), to=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/.tmp/B/8c1a458ee4d44b2d867154a3a14ea740 2024-11-18T20:14:13,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/.tmp/C/dd74de3ef0734a72a3e6043697fb745e is 50, key is test_row_0/C:col10/1731960851469/Put/seqid=0 2024-11-18T20:14:13,225 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742310_1486 (size=12001) 2024-11-18T20:14:13,225 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=126 (bloomFilter=true), to=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/.tmp/C/dd74de3ef0734a72a3e6043697fb745e 2024-11-18T20:14:13,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/.tmp/A/f3862c91776c40a48b22b8344ee618e4 as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/A/f3862c91776c40a48b22b8344ee618e4 2024-11-18T20:14:13,232 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/A/f3862c91776c40a48b22b8344ee618e4, entries=150, sequenceid=126, filesize=11.7 K 2024-11-18T20:14:13,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/.tmp/B/8c1a458ee4d44b2d867154a3a14ea740 as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/B/8c1a458ee4d44b2d867154a3a14ea740 2024-11-18T20:14:13,236 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/B/8c1a458ee4d44b2d867154a3a14ea740, entries=150, sequenceid=126, filesize=11.7 K 2024-11-18T20:14:13,237 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/.tmp/C/dd74de3ef0734a72a3e6043697fb745e as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/C/dd74de3ef0734a72a3e6043697fb745e 2024-11-18T20:14:13,244 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/C/dd74de3ef0734a72a3e6043697fb745e, entries=150, sequenceid=126, filesize=11.7 K 2024-11-18T20:14:13,245 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 23d90021d1eda01703e7bdf7270f74fc in 852ms, sequenceid=126, compaction requested=false 2024-11-18T20:14:13,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2603): Flush status journal for 23d90021d1eda01703e7bdf7270f74fc: 2024-11-18T20:14:13,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc. 2024-11-18T20:14:13,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=126 2024-11-18T20:14:13,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.HMaster(4169): Remote procedure done, pid=126 2024-11-18T20:14:13,247 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=126, resume processing ppid=125 2024-11-18T20:14:13,247 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=126, ppid=125, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9210 sec 2024-11-18T20:14:13,248 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=125, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees in 1.9250 sec 2024-11-18T20:14:13,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(8855): Flush requested on 23d90021d1eda01703e7bdf7270f74fc 2024-11-18T20:14:13,299 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-18T20:14:13,299 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 23d90021d1eda01703e7bdf7270f74fc 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-18T20:14:13,302 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/.tmp/A/f626567b5b7146e3aca8211a8e562fb7 is 50, key is test_row_0/A:col10/1731960852659/Put/seqid=0 2024-11-18T20:14:13,305 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742311_1487 (size=14541) 2024-11-18T20:14:13,309 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=155 (bloomFilter=true), to=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/.tmp/A/f626567b5b7146e3aca8211a8e562fb7 2024-11-18T20:14:13,315 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/.tmp/B/5588fa102d214bf5bc1720d27c364e70 is 50, key is test_row_0/B:col10/1731960852659/Put/seqid=0 2024-11-18T20:14:13,319 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742312_1488 (size=12151) 2024-11-18T20:14:13,329 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:13,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51862 deadline: 1731960913322, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:13,329 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:13,329 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:13,329 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:14:13,332 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:13,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51924 deadline: 1731960913323, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:13,332 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:13,332 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:13,332 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:14:13,332 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:13,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51878 deadline: 1731960913323, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:13,333 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:13,333 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:13,333 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:13,333 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:14:13,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51894 deadline: 1731960913323, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:13,333 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:13,333 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:13,333 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:13,333 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:14:13,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51860 deadline: 1731960913329, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:13,333 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:13,333 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:13,333 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:14:13,435 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:13,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51862 deadline: 1731960913433, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:13,436 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:13,436 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:13,436 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:14:13,448 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:13,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51860 deadline: 1731960913443, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:13,449 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:13,449 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:13,449 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:14:13,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-18T20:14:13,473 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-18T20:14:13,473 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-18T20:14:13,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] procedure2.ProcedureExecutor(1139): Stored pid=127, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees 2024-11-18T20:14:13,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-18T20:14:13,475 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=127, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-18T20:14:13,475 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=127, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-18T20:14:13,475 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=128, ppid=127, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-18T20:14:13,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-18T20:14:13,627 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35963 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-18T20:14:13,627 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc. 2024-11-18T20:14:13,627 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc. as already flushing 2024-11-18T20:14:13,627 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc. 2024-11-18T20:14:13,627 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 23d90021d1eda01703e7bdf7270f74fc, NAME => 'TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:14:13,627 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 23d90021d1eda01703e7bdf7270f74fc, NAME => 'TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:14:13,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 23d90021d1eda01703e7bdf7270f74fc, NAME => 'TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 23d90021d1eda01703e7bdf7270f74fc, NAME => 'TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:14:13,648 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:13,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51862 deadline: 1731960913643, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:13,648 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:13,648 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:13,648 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:14:13,658 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:13,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51860 deadline: 1731960913653, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:13,658 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:13,658 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:13,658 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:14:13,720 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=155 (bloomFilter=true), to=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/.tmp/B/5588fa102d214bf5bc1720d27c364e70 2024-11-18T20:14:13,726 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/.tmp/C/f79bc7fc9d474d2d85d75f490c15b551 is 50, key is test_row_0/C:col10/1731960852659/Put/seqid=0 2024-11-18T20:14:13,729 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742313_1489 (size=12151) 2024-11-18T20:14:13,779 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35963 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-18T20:14:13,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc. 2024-11-18T20:14:13,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc. as already flushing 2024-11-18T20:14:13,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc. 2024-11-18T20:14:13,780 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 23d90021d1eda01703e7bdf7270f74fc, NAME => 'TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:14:13,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 23d90021d1eda01703e7bdf7270f74fc, NAME => 'TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:14:13,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 23d90021d1eda01703e7bdf7270f74fc, NAME => 'TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 23d90021d1eda01703e7bdf7270f74fc, NAME => 'TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:14:13,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-18T20:14:13,847 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:13,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51924 deadline: 1731960913843, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:13,847 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:13,847 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:13,847 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:13,847 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:14:13,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51878 deadline: 1731960913843, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:13,847 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:13,847 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:13,847 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:14:13,847 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:13,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51894 deadline: 1731960913843, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:13,848 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:13,848 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:13,848 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:14:13,931 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35963 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-18T20:14:13,931 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc. 2024-11-18T20:14:13,931 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc. as already flushing 2024-11-18T20:14:13,931 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc. 2024-11-18T20:14:13,932 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 23d90021d1eda01703e7bdf7270f74fc, NAME => 'TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:14:13,932 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 23d90021d1eda01703e7bdf7270f74fc, NAME => 'TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:14:13,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 23d90021d1eda01703e7bdf7270f74fc, NAME => 'TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 23d90021d1eda01703e7bdf7270f74fc, NAME => 'TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:14:13,959 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:13,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51862 deadline: 1731960913953, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:13,960 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:13,960 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:13,960 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:14:13,968 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:13,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51860 deadline: 1731960913963, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:13,969 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:13,969 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:13,969 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:14:14,084 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35963 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-18T20:14:14,084 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc. 2024-11-18T20:14:14,084 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc. as already flushing 2024-11-18T20:14:14,084 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc. 2024-11-18T20:14:14,084 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 23d90021d1eda01703e7bdf7270f74fc, NAME => 'TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:14:14,084 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 23d90021d1eda01703e7bdf7270f74fc, NAME => 'TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:14:14,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 23d90021d1eda01703e7bdf7270f74fc, NAME => 'TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 23d90021d1eda01703e7bdf7270f74fc, NAME => 'TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:14:14,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-18T20:14:14,129 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=155 (bloomFilter=true), to=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/.tmp/C/f79bc7fc9d474d2d85d75f490c15b551 2024-11-18T20:14:14,132 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/.tmp/A/f626567b5b7146e3aca8211a8e562fb7 as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/A/f626567b5b7146e3aca8211a8e562fb7 2024-11-18T20:14:14,135 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/A/f626567b5b7146e3aca8211a8e562fb7, entries=200, sequenceid=155, filesize=14.2 K 2024-11-18T20:14:14,135 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/.tmp/B/5588fa102d214bf5bc1720d27c364e70 as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/B/5588fa102d214bf5bc1720d27c364e70 2024-11-18T20:14:14,138 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/B/5588fa102d214bf5bc1720d27c364e70, entries=150, sequenceid=155, filesize=11.9 K 2024-11-18T20:14:14,139 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/.tmp/C/f79bc7fc9d474d2d85d75f490c15b551 as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/C/f79bc7fc9d474d2d85d75f490c15b551 2024-11-18T20:14:14,141 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/C/f79bc7fc9d474d2d85d75f490c15b551, entries=150, sequenceid=155, filesize=11.9 K 2024-11-18T20:14:14,142 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for 23d90021d1eda01703e7bdf7270f74fc in 843ms, sequenceid=155, compaction requested=true 2024-11-18T20:14:14,142 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 23d90021d1eda01703e7bdf7270f74fc: 2024-11-18T20:14:14,142 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 23d90021d1eda01703e7bdf7270f74fc:A, priority=-2147483648, current under compaction store size is 1 2024-11-18T20:14:14,142 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:14:14,142 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T20:14:14,142 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 23d90021d1eda01703e7bdf7270f74fc:B, priority=-2147483648, current under compaction store size is 2 2024-11-18T20:14:14,142 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T20:14:14,142 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 23d90021d1eda01703e7bdf7270f74fc:C, priority=-2147483648, current under compaction store size is 3 2024-11-18T20:14:14,142 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T20:14:14,142 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T20:14:14,143 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38783 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T20:14:14,143 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36393 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T20:14:14,143 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HStore(1541): 23d90021d1eda01703e7bdf7270f74fc/B is initiating minor compaction (all files) 2024-11-18T20:14:14,143 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HStore(1541): 23d90021d1eda01703e7bdf7270f74fc/A is initiating minor compaction (all files) 2024-11-18T20:14:14,144 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 23d90021d1eda01703e7bdf7270f74fc/B in TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc. 2024-11-18T20:14:14,144 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 23d90021d1eda01703e7bdf7270f74fc/A in TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc. 2024-11-18T20:14:14,144 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/B/d40c2f043c674d86808c3bbc74972a1e, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/B/8c1a458ee4d44b2d867154a3a14ea740, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/B/5588fa102d214bf5bc1720d27c364e70] into tmpdir=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/.tmp, totalSize=35.5 K 2024-11-18T20:14:14,144 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/A/0de15de0a61b4d81b7cc2edfcfeb51c0, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/A/f3862c91776c40a48b22b8344ee618e4, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/A/f626567b5b7146e3aca8211a8e562fb7] into tmpdir=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/.tmp, totalSize=37.9 K 2024-11-18T20:14:14,144 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.Compactor(225): Compacting 0de15de0a61b4d81b7cc2edfcfeb51c0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=115, earliestPutTs=1731960850800 2024-11-18T20:14:14,144 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] compactions.Compactor(225): Compacting d40c2f043c674d86808c3bbc74972a1e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=115, earliestPutTs=1731960850800 2024-11-18T20:14:14,144 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8c1a458ee4d44b2d867154a3a14ea740, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=126, earliestPutTs=1731960851461 2024-11-18T20:14:14,144 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.Compactor(225): Compacting f3862c91776c40a48b22b8344ee618e4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=126, earliestPutTs=1731960851461 2024-11-18T20:14:14,144 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.Compactor(225): Compacting f626567b5b7146e3aca8211a8e562fb7, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1731960852659 2024-11-18T20:14:14,144 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5588fa102d214bf5bc1720d27c364e70, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1731960852659 2024-11-18T20:14:14,152 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 23d90021d1eda01703e7bdf7270f74fc#B#compaction#413 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:14:14,152 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/.tmp/B/c1a2b0e2648f48858e17aaaef84ce55f is 50, key is test_row_0/B:col10/1731960852659/Put/seqid=0 2024-11-18T20:14:14,158 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 23d90021d1eda01703e7bdf7270f74fc#A#compaction#414 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:14:14,159 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/.tmp/A/aa9fb2625a694b76a14c4815eacac20e is 50, key is test_row_0/A:col10/1731960852659/Put/seqid=0 2024-11-18T20:14:14,167 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742314_1490 (size=12493) 2024-11-18T20:14:14,171 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742315_1491 (size=12493) 2024-11-18T20:14:14,236 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35963 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-18T20:14:14,236 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc. 2024-11-18T20:14:14,237 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2902): Flushing 23d90021d1eda01703e7bdf7270f74fc 3/3 column families, dataSize=46.96 KB heapSize=123.80 KB 2024-11-18T20:14:14,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/.tmp/A/aa58adf38ed346578f5068ed30c58a60 is 50, key is test_row_0/A:col10/1731960853328/Put/seqid=0 2024-11-18T20:14:14,254 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742316_1492 (size=12151) 2024-11-18T20:14:14,255 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=165 (bloomFilter=true), to=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/.tmp/A/aa58adf38ed346578f5068ed30c58a60 2024-11-18T20:14:14,262 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/.tmp/B/1c7bea34baa14059b8bcf482b1f42d3d is 50, key is test_row_0/B:col10/1731960853328/Put/seqid=0 2024-11-18T20:14:14,273 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742317_1493 (size=12151) 2024-11-18T20:14:14,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(8855): Flush requested on 23d90021d1eda01703e7bdf7270f74fc 2024-11-18T20:14:14,466 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc. as already flushing 2024-11-18T20:14:14,537 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:14,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51860 deadline: 1731960914534, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:14,537 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:14,537 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:14,537 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:14:14,542 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:14,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51862 deadline: 1731960914535, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:14,542 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:14,542 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:14,542 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:14:14,570 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/.tmp/B/c1a2b0e2648f48858e17aaaef84ce55f as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/B/c1a2b0e2648f48858e17aaaef84ce55f 2024-11-18T20:14:14,574 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 23d90021d1eda01703e7bdf7270f74fc/B of 23d90021d1eda01703e7bdf7270f74fc into c1a2b0e2648f48858e17aaaef84ce55f(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:14:14,574 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 23d90021d1eda01703e7bdf7270f74fc: 2024-11-18T20:14:14,574 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., storeName=23d90021d1eda01703e7bdf7270f74fc/B, priority=13, startTime=1731960854142; duration=0sec 2024-11-18T20:14:14,574 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T20:14:14,574 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 23d90021d1eda01703e7bdf7270f74fc:B 2024-11-18T20:14:14,574 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/.tmp/A/aa9fb2625a694b76a14c4815eacac20e as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/A/aa9fb2625a694b76a14c4815eacac20e 2024-11-18T20:14:14,574 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T20:14:14,575 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36393 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T20:14:14,575 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HStore(1541): 23d90021d1eda01703e7bdf7270f74fc/C is initiating minor compaction (all files) 2024-11-18T20:14:14,575 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 23d90021d1eda01703e7bdf7270f74fc/C in TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc. 2024-11-18T20:14:14,575 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/C/ded89bf64a10475ca371570944ef60a9, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/C/dd74de3ef0734a72a3e6043697fb745e, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/C/f79bc7fc9d474d2d85d75f490c15b551] into tmpdir=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/.tmp, totalSize=35.5 K 2024-11-18T20:14:14,576 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] compactions.Compactor(225): Compacting ded89bf64a10475ca371570944ef60a9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=115, earliestPutTs=1731960850800 2024-11-18T20:14:14,577 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] compactions.Compactor(225): Compacting dd74de3ef0734a72a3e6043697fb745e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=126, earliestPutTs=1731960851461 2024-11-18T20:14:14,577 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] compactions.Compactor(225): Compacting f79bc7fc9d474d2d85d75f490c15b551, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1731960852659 2024-11-18T20:14:14,579 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 23d90021d1eda01703e7bdf7270f74fc/A of 23d90021d1eda01703e7bdf7270f74fc into aa9fb2625a694b76a14c4815eacac20e(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:14:14,580 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 23d90021d1eda01703e7bdf7270f74fc: 2024-11-18T20:14:14,580 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., storeName=23d90021d1eda01703e7bdf7270f74fc/A, priority=13, startTime=1731960854142; duration=0sec 2024-11-18T20:14:14,580 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:14:14,580 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 23d90021d1eda01703e7bdf7270f74fc:A 2024-11-18T20:14:14,583 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 23d90021d1eda01703e7bdf7270f74fc#C#compaction#417 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:14:14,584 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/.tmp/C/2ff61129268341d3a3b634a2a6eb686b is 50, key is test_row_0/C:col10/1731960852659/Put/seqid=0 2024-11-18T20:14:14,588 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742318_1494 (size=12493) 2024-11-18T20:14:14,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-18T20:14:14,647 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:14,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51860 deadline: 1731960914643, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:14,648 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:14,648 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:14,648 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:14:14,656 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:14,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51862 deadline: 1731960914653, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:14,656 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:14,656 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:14,656 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:14:14,674 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=165 (bloomFilter=true), to=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/.tmp/B/1c7bea34baa14059b8bcf482b1f42d3d 2024-11-18T20:14:14,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/.tmp/C/90eb122cce754bfab78929f91785afeb is 50, key is test_row_0/C:col10/1731960853328/Put/seqid=0 2024-11-18T20:14:14,684 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742319_1495 (size=12151) 2024-11-18T20:14:14,858 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:14,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51860 deadline: 1731960914853, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:14,859 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:14,859 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:14,859 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:14:14,859 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:14,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51894 deadline: 1731960914853, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:14,859 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to c0a89b2656d4,35963,1731960718223 for 'test_row_1' in 23d90021d1eda01703e7bdf7270f74fc of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2192 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-18T20:14:14,860 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:14,860 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:14,860 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:14:14,868 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:14,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51924 deadline: 1731960914862, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:14,868 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:14,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51878 deadline: 1731960914863, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:14,869 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:14,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51862 deadline: 1731960914863, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:14,869 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to c0a89b2656d4,35963,1731960718223 for 'test_row_1' in 23d90021d1eda01703e7bdf7270f74fc of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2208 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-18T20:14:14,869 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to c0a89b2656d4,35963,1731960718223 for 'test_row_0' in 23d90021d1eda01703e7bdf7270f74fc of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2202 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-18T20:14:14,869 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:14,869 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:14,869 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:14,869 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:14:14,869 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:14,869 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:14:14,869 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:14,869 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:14,870 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:14:14,991 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/.tmp/C/2ff61129268341d3a3b634a2a6eb686b as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/C/2ff61129268341d3a3b634a2a6eb686b 2024-11-18T20:14:14,995 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 23d90021d1eda01703e7bdf7270f74fc/C of 23d90021d1eda01703e7bdf7270f74fc into 2ff61129268341d3a3b634a2a6eb686b(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:14:14,995 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 23d90021d1eda01703e7bdf7270f74fc: 2024-11-18T20:14:14,995 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., storeName=23d90021d1eda01703e7bdf7270f74fc/C, priority=13, startTime=1731960854142; duration=0sec 2024-11-18T20:14:14,995 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:14:14,995 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 23d90021d1eda01703e7bdf7270f74fc:C 2024-11-18T20:14:15,084 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=165 (bloomFilter=true), to=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/.tmp/C/90eb122cce754bfab78929f91785afeb 2024-11-18T20:14:15,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/.tmp/A/aa58adf38ed346578f5068ed30c58a60 as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/A/aa58adf38ed346578f5068ed30c58a60 2024-11-18T20:14:15,090 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/A/aa58adf38ed346578f5068ed30c58a60, entries=150, sequenceid=165, filesize=11.9 K 2024-11-18T20:14:15,091 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/.tmp/B/1c7bea34baa14059b8bcf482b1f42d3d as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/B/1c7bea34baa14059b8bcf482b1f42d3d 2024-11-18T20:14:15,095 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/B/1c7bea34baa14059b8bcf482b1f42d3d, entries=150, sequenceid=165, filesize=11.9 K 2024-11-18T20:14:15,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/.tmp/C/90eb122cce754bfab78929f91785afeb as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/C/90eb122cce754bfab78929f91785afeb 2024-11-18T20:14:15,098 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/C/90eb122cce754bfab78929f91785afeb, entries=150, sequenceid=165, filesize=11.9 K 2024-11-18T20:14:15,098 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~123.75 KB/126720, currentSize=154.31 KB/158010 for 23d90021d1eda01703e7bdf7270f74fc in 861ms, sequenceid=165, compaction requested=false 2024-11-18T20:14:15,098 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2603): Flush status journal for 23d90021d1eda01703e7bdf7270f74fc: 2024-11-18T20:14:15,098 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc. 2024-11-18T20:14:15,098 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=128 2024-11-18T20:14:15,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.HMaster(4169): Remote procedure done, pid=128 2024-11-18T20:14:15,102 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=128, resume processing ppid=127 2024-11-18T20:14:15,102 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=128, ppid=127, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6240 sec 2024-11-18T20:14:15,104 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=127, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees in 1.6290 sec 2024-11-18T20:14:15,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(8855): Flush requested on 23d90021d1eda01703e7bdf7270f74fc 2024-11-18T20:14:15,166 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-18T20:14:15,166 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 23d90021d1eda01703e7bdf7270f74fc 3/3 column families, dataSize=161.02 KB heapSize=422.63 KB 2024-11-18T20:14:15,171 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/.tmp/A/c3532bf789364ac99a0e6849f7c6385b is 50, key is test_row_0/A:col10/1731960854532/Put/seqid=0 2024-11-18T20:14:15,186 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742320_1496 (size=14541) 2024-11-18T20:14:15,198 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:15,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51862 deadline: 1731960915192, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:15,199 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:15,199 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:15,199 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:14:15,199 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:15,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51860 deadline: 1731960915193, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:15,200 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:15,200 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:15,200 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:14:15,307 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:15,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51862 deadline: 1731960915303, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:15,307 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:15,307 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:15,307 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:15,307 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:14:15,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51860 deadline: 1731960915303, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:15,308 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:15,308 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:15,308 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:14:15,513 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:15,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51862 deadline: 1731960915513, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:15,514 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:15,514 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:15,514 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:15,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51860 deadline: 1731960915513, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:15,514 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:14:15,514 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:15,514 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:15,514 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:14:15,586 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=195 (bloomFilter=true), to=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/.tmp/A/c3532bf789364ac99a0e6849f7c6385b 2024-11-18T20:14:15,593 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/.tmp/B/df3a12917687463cbf9ba225022783bc is 50, key is test_row_0/B:col10/1731960854532/Put/seqid=0 2024-11-18T20:14:15,597 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742321_1497 (size=12151) 2024-11-18T20:14:15,597 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=195 (bloomFilter=true), to=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/.tmp/B/df3a12917687463cbf9ba225022783bc 2024-11-18T20:14:15,607 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/.tmp/C/9104fa5f16e54055935b21e2eeaffa45 is 50, key is test_row_0/C:col10/1731960854532/Put/seqid=0 2024-11-18T20:14:15,610 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742322_1498 (size=12151) 2024-11-18T20:14:15,610 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=195 (bloomFilter=true), to=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/.tmp/C/9104fa5f16e54055935b21e2eeaffa45 2024-11-18T20:14:15,616 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/.tmp/A/c3532bf789364ac99a0e6849f7c6385b as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/A/c3532bf789364ac99a0e6849f7c6385b 2024-11-18T20:14:15,620 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/A/c3532bf789364ac99a0e6849f7c6385b, entries=200, sequenceid=195, filesize=14.2 K 2024-11-18T20:14:15,621 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/.tmp/B/df3a12917687463cbf9ba225022783bc as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/B/df3a12917687463cbf9ba225022783bc 2024-11-18T20:14:15,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-18T20:14:15,623 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-18T20:14:15,624 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-18T20:14:15,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] procedure2.ProcedureExecutor(1139): Stored pid=129, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees 2024-11-18T20:14:15,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-18T20:14:15,627 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=129, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-18T20:14:15,627 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=129, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-18T20:14:15,627 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=130, ppid=129, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-18T20:14:15,632 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/B/df3a12917687463cbf9ba225022783bc, entries=150, sequenceid=195, filesize=11.9 K 2024-11-18T20:14:15,633 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/.tmp/C/9104fa5f16e54055935b21e2eeaffa45 as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/C/9104fa5f16e54055935b21e2eeaffa45 2024-11-18T20:14:15,636 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/C/9104fa5f16e54055935b21e2eeaffa45, entries=150, sequenceid=195, filesize=11.9 K 2024-11-18T20:14:15,637 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~422.58 KB/432720, currentSize=40.25 KB/41220 for 23d90021d1eda01703e7bdf7270f74fc in 471ms, sequenceid=195, compaction requested=true 2024-11-18T20:14:15,637 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 23d90021d1eda01703e7bdf7270f74fc: 2024-11-18T20:14:15,637 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 23d90021d1eda01703e7bdf7270f74fc:A, priority=-2147483648, current under compaction store size is 1 2024-11-18T20:14:15,637 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:14:15,637 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T20:14:15,637 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 23d90021d1eda01703e7bdf7270f74fc:B, priority=-2147483648, current under compaction store size is 2 2024-11-18T20:14:15,637 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:14:15,637 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T20:14:15,637 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 23d90021d1eda01703e7bdf7270f74fc:C, priority=-2147483648, current under compaction store size is 3 2024-11-18T20:14:15,637 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T20:14:15,638 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T20:14:15,638 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HStore(1541): 23d90021d1eda01703e7bdf7270f74fc/B is initiating minor compaction (all files) 2024-11-18T20:14:15,638 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 23d90021d1eda01703e7bdf7270f74fc/B in TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc. 2024-11-18T20:14:15,638 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39185 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T20:14:15,638 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HStore(1541): 23d90021d1eda01703e7bdf7270f74fc/A is initiating minor compaction (all files) 2024-11-18T20:14:15,638 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/B/c1a2b0e2648f48858e17aaaef84ce55f, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/B/1c7bea34baa14059b8bcf482b1f42d3d, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/B/df3a12917687463cbf9ba225022783bc] into tmpdir=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/.tmp, totalSize=35.9 K 2024-11-18T20:14:15,638 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 23d90021d1eda01703e7bdf7270f74fc/A in TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc. 2024-11-18T20:14:15,638 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/A/aa9fb2625a694b76a14c4815eacac20e, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/A/aa58adf38ed346578f5068ed30c58a60, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/A/c3532bf789364ac99a0e6849f7c6385b] into tmpdir=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/.tmp, totalSize=38.3 K 2024-11-18T20:14:15,638 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.Compactor(225): Compacting c1a2b0e2648f48858e17aaaef84ce55f, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1731960852659 2024-11-18T20:14:15,638 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] compactions.Compactor(225): Compacting aa9fb2625a694b76a14c4815eacac20e, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1731960852659 2024-11-18T20:14:15,639 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.Compactor(225): Compacting 1c7bea34baa14059b8bcf482b1f42d3d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=165, earliestPutTs=1731960853319 2024-11-18T20:14:15,639 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] compactions.Compactor(225): Compacting aa58adf38ed346578f5068ed30c58a60, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=165, earliestPutTs=1731960853319 2024-11-18T20:14:15,639 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.Compactor(225): Compacting df3a12917687463cbf9ba225022783bc, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1731960854523 2024-11-18T20:14:15,639 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] compactions.Compactor(225): Compacting c3532bf789364ac99a0e6849f7c6385b, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1731960854523 2024-11-18T20:14:15,645 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 23d90021d1eda01703e7bdf7270f74fc#B#compaction#422 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:14:15,645 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/.tmp/B/ba25119d8ed745419b3bb21168a659c5 is 50, key is test_row_0/B:col10/1731960854532/Put/seqid=0 2024-11-18T20:14:15,650 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 23d90021d1eda01703e7bdf7270f74fc#A#compaction#423 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:14:15,650 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/.tmp/A/eca3d1d04c9d47dc81c13429d7a59988 is 50, key is test_row_0/A:col10/1731960854532/Put/seqid=0 2024-11-18T20:14:15,656 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742323_1499 (size=12595) 2024-11-18T20:14:15,658 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742324_1500 (size=12595) 2024-11-18T20:14:15,664 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/.tmp/B/ba25119d8ed745419b3bb21168a659c5 as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/B/ba25119d8ed745419b3bb21168a659c5 2024-11-18T20:14:15,669 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 23d90021d1eda01703e7bdf7270f74fc/B of 23d90021d1eda01703e7bdf7270f74fc into ba25119d8ed745419b3bb21168a659c5(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:14:15,669 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 23d90021d1eda01703e7bdf7270f74fc: 2024-11-18T20:14:15,669 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., storeName=23d90021d1eda01703e7bdf7270f74fc/B, priority=13, startTime=1731960855637; duration=0sec 2024-11-18T20:14:15,669 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T20:14:15,669 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 23d90021d1eda01703e7bdf7270f74fc:B 2024-11-18T20:14:15,669 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T20:14:15,670 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T20:14:15,670 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HStore(1541): 23d90021d1eda01703e7bdf7270f74fc/C is initiating minor compaction (all files) 2024-11-18T20:14:15,670 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 23d90021d1eda01703e7bdf7270f74fc/C in TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc. 2024-11-18T20:14:15,670 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/C/2ff61129268341d3a3b634a2a6eb686b, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/C/90eb122cce754bfab78929f91785afeb, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/C/9104fa5f16e54055935b21e2eeaffa45] into tmpdir=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/.tmp, totalSize=35.9 K 2024-11-18T20:14:15,671 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.Compactor(225): Compacting 2ff61129268341d3a3b634a2a6eb686b, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1731960852659 2024-11-18T20:14:15,671 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.Compactor(225): Compacting 90eb122cce754bfab78929f91785afeb, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=165, earliestPutTs=1731960853319 2024-11-18T20:14:15,671 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.Compactor(225): Compacting 9104fa5f16e54055935b21e2eeaffa45, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1731960854523 2024-11-18T20:14:15,678 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 23d90021d1eda01703e7bdf7270f74fc#C#compaction#424 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:14:15,679 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/.tmp/C/b1e55172024f4005af135a2cd54f72d8 is 50, key is test_row_0/C:col10/1731960854532/Put/seqid=0 2024-11-18T20:14:15,686 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742325_1501 (size=12595) 2024-11-18T20:14:15,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-18T20:14:15,781 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35963 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-18T20:14:15,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc. 2024-11-18T20:14:15,782 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2902): Flushing 23d90021d1eda01703e7bdf7270f74fc 3/3 column families, dataSize=40.25 KB heapSize=106.22 KB 2024-11-18T20:14:15,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/.tmp/A/b21202671a9743f282ebd8a418b05a2f is 50, key is test_row_0/A:col10/1731960855190/Put/seqid=0 2024-11-18T20:14:15,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(8855): Flush requested on 23d90021d1eda01703e7bdf7270f74fc 2024-11-18T20:14:15,827 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc. as already flushing 2024-11-18T20:14:15,830 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742326_1502 (size=12151) 2024-11-18T20:14:15,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-18T20:14:15,969 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:15,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51860 deadline: 1731960915961, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:15,969 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:15,969 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:15,969 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:14:15,974 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:15,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51862 deadline: 1731960915962, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:15,975 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:15,975 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:15,975 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:14:16,063 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/.tmp/A/eca3d1d04c9d47dc81c13429d7a59988 as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/A/eca3d1d04c9d47dc81c13429d7a59988 2024-11-18T20:14:16,067 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 23d90021d1eda01703e7bdf7270f74fc/A of 23d90021d1eda01703e7bdf7270f74fc into eca3d1d04c9d47dc81c13429d7a59988(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:14:16,067 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 23d90021d1eda01703e7bdf7270f74fc: 2024-11-18T20:14:16,067 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., storeName=23d90021d1eda01703e7bdf7270f74fc/A, priority=13, startTime=1731960855637; duration=0sec 2024-11-18T20:14:16,067 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:14:16,067 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 23d90021d1eda01703e7bdf7270f74fc:A 2024-11-18T20:14:16,080 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:16,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51860 deadline: 1731960916072, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:16,082 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:16,082 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:16,082 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:14:16,090 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:16,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51862 deadline: 1731960916083, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:16,091 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/.tmp/C/b1e55172024f4005af135a2cd54f72d8 as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/C/b1e55172024f4005af135a2cd54f72d8 2024-11-18T20:14:16,092 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:16,092 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:16,093 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:14:16,096 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 23d90021d1eda01703e7bdf7270f74fc/C of 23d90021d1eda01703e7bdf7270f74fc into b1e55172024f4005af135a2cd54f72d8(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:14:16,096 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 23d90021d1eda01703e7bdf7270f74fc: 2024-11-18T20:14:16,096 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., storeName=23d90021d1eda01703e7bdf7270f74fc/C, priority=13, startTime=1731960855637; duration=0sec 2024-11-18T20:14:16,096 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:14:16,096 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 23d90021d1eda01703e7bdf7270f74fc:C 2024-11-18T20:14:16,231 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=205 (bloomFilter=true), to=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/.tmp/A/b21202671a9743f282ebd8a418b05a2f 2024-11-18T20:14:16,238 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/.tmp/B/20c2c0e6b3c7463781ad28c4435ce89b is 50, key is test_row_0/B:col10/1731960855190/Put/seqid=0 2024-11-18T20:14:16,242 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742327_1503 (size=12151) 2024-11-18T20:14:16,243 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=205 (bloomFilter=true), to=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/.tmp/B/20c2c0e6b3c7463781ad28c4435ce89b 2024-11-18T20:14:16,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-18T20:14:16,253 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/.tmp/C/c833eb9b468546dbb7f321cdf01d8767 is 50, key is test_row_0/C:col10/1731960855190/Put/seqid=0 2024-11-18T20:14:16,257 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742328_1504 (size=12151) 2024-11-18T20:14:16,294 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:16,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51860 deadline: 1731960916293, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:16,294 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:16,294 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:16,294 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:14:16,309 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:16,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51862 deadline: 1731960916304, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:16,312 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:16,312 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:16,312 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:14:16,605 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:16,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51860 deadline: 1731960916603, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:16,605 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:16,605 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:16,605 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:14:16,625 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:16,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51862 deadline: 1731960916623, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:16,625 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:16,625 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:16,625 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:14:16,657 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=205 (bloomFilter=true), to=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/.tmp/C/c833eb9b468546dbb7f321cdf01d8767 2024-11-18T20:14:16,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/.tmp/A/b21202671a9743f282ebd8a418b05a2f as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/A/b21202671a9743f282ebd8a418b05a2f 2024-11-18T20:14:16,664 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/A/b21202671a9743f282ebd8a418b05a2f, entries=150, sequenceid=205, filesize=11.9 K 2024-11-18T20:14:16,665 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/.tmp/B/20c2c0e6b3c7463781ad28c4435ce89b as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/B/20c2c0e6b3c7463781ad28c4435ce89b 2024-11-18T20:14:16,674 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/B/20c2c0e6b3c7463781ad28c4435ce89b, entries=150, sequenceid=205, filesize=11.9 K 2024-11-18T20:14:16,675 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/.tmp/C/c833eb9b468546dbb7f321cdf01d8767 as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/C/c833eb9b468546dbb7f321cdf01d8767 2024-11-18T20:14:16,679 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/C/c833eb9b468546dbb7f321cdf01d8767, entries=150, sequenceid=205, filesize=11.9 K 2024-11-18T20:14:16,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.StoreScanner(1005): StoreScanner already closing. There is no need to updateReaders 2024-11-18T20:14:16,681 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~106.17 KB/108720, currentSize=161.02 KB/164880 for 23d90021d1eda01703e7bdf7270f74fc in 899ms, sequenceid=205, compaction requested=false 2024-11-18T20:14:16,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2603): Flush status journal for 23d90021d1eda01703e7bdf7270f74fc: 2024-11-18T20:14:16,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc. 2024-11-18T20:14:16,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=130 2024-11-18T20:14:16,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.HMaster(4169): Remote procedure done, pid=130 2024-11-18T20:14:16,684 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=130, resume processing ppid=129 2024-11-18T20:14:16,684 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=130, ppid=129, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0550 sec 2024-11-18T20:14:16,685 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=129, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees in 1.0600 sec 2024-11-18T20:14:16,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-18T20:14:16,763 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-18T20:14:16,764 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-18T20:14:16,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] procedure2.ProcedureExecutor(1139): Stored pid=131, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees 2024-11-18T20:14:16,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-18T20:14:16,767 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=131, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-18T20:14:16,767 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=131, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-18T20:14:16,767 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=132, ppid=131, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-18T20:14:16,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-18T20:14:16,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(8855): Flush requested on 23d90021d1eda01703e7bdf7270f74fc 2024-11-18T20:14:16,880 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-18T20:14:16,880 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 23d90021d1eda01703e7bdf7270f74fc 3/3 column families, dataSize=167.72 KB heapSize=440.20 KB 2024-11-18T20:14:16,885 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/.tmp/A/b4a146acc8b14703a2150a4f74d4d8b8 is 50, key is test_row_0/A:col10/1731960855944/Put/seqid=0 2024-11-18T20:14:16,888 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742329_1505 (size=14541) 2024-11-18T20:14:16,906 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:16,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51878 deadline: 1731960916901, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:16,906 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:16,906 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:16,906 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:14:16,912 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:16,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51894 deadline: 1731960916903, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:16,913 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:16,913 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:16,913 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:16,913 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:14:16,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51924 deadline: 1731960916903, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:16,913 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:16,913 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:16,913 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:14:16,920 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35963 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-18T20:14:16,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc. 2024-11-18T20:14:16,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc. as already flushing 2024-11-18T20:14:16,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc. 2024-11-18T20:14:16,920 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 23d90021d1eda01703e7bdf7270f74fc, NAME => 'TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:14:16,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 23d90021d1eda01703e7bdf7270f74fc, NAME => 'TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:14:16,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 23d90021d1eda01703e7bdf7270f74fc, NAME => 'TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 23d90021d1eda01703e7bdf7270f74fc, NAME => 'TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:14:17,019 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:17,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51878 deadline: 1731960917013, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:17,019 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:17,019 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:17,019 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:14:17,030 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:17,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51894 deadline: 1731960917023, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:17,030 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:17,030 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:17,030 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:17,030 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:14:17,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51924 deadline: 1731960917023, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:17,031 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:17,031 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:17,031 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:14:17,071 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35963 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-18T20:14:17,071 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc. 2024-11-18T20:14:17,071 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc. as already flushing 2024-11-18T20:14:17,072 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc. 2024-11-18T20:14:17,072 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 23d90021d1eda01703e7bdf7270f74fc, NAME => 'TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:14:17,072 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 23d90021d1eda01703e7bdf7270f74fc, NAME => 'TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:14:17,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 23d90021d1eda01703e7bdf7270f74fc, NAME => 'TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 23d90021d1eda01703e7bdf7270f74fc, NAME => 'TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:14:17,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-18T20:14:17,117 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:17,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51860 deadline: 1731960917113, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:17,117 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:17,117 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:17,117 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:14:17,137 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:17,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51862 deadline: 1731960917133, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:17,138 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:17,138 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:17,138 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:14:17,224 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35963 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-18T20:14:17,224 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc. 2024-11-18T20:14:17,224 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc. as already flushing 2024-11-18T20:14:17,224 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc. 2024-11-18T20:14:17,224 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 23d90021d1eda01703e7bdf7270f74fc, NAME => 'TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:14:17,224 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 23d90021d1eda01703e7bdf7270f74fc, NAME => 'TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:14:17,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 23d90021d1eda01703e7bdf7270f74fc, NAME => 'TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 23d90021d1eda01703e7bdf7270f74fc, NAME => 'TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:14:17,230 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:17,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51878 deadline: 1731960917223, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:17,230 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:17,230 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:17,230 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:14:17,238 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:17,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51894 deadline: 1731960917233, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:17,239 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:17,239 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:17,239 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:14:17,247 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:17,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51924 deadline: 1731960917243, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:17,248 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:17,248 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:17,248 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:14:17,289 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=235 (bloomFilter=true), to=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/.tmp/A/b4a146acc8b14703a2150a4f74d4d8b8 2024-11-18T20:14:17,295 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/.tmp/B/f04cc9ecbf224d8791ec076ae5a8292a is 50, key is test_row_0/B:col10/1731960855944/Put/seqid=0 2024-11-18T20:14:17,302 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742330_1506 (size=12151) 2024-11-18T20:14:17,304 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=235 (bloomFilter=true), to=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/.tmp/B/f04cc9ecbf224d8791ec076ae5a8292a 2024-11-18T20:14:17,310 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/.tmp/C/2df50040410a49a0ba45e6139fdd1a88 is 50, key is test_row_0/C:col10/1731960855944/Put/seqid=0 2024-11-18T20:14:17,315 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742331_1507 (size=12151) 2024-11-18T20:14:17,377 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35963 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-18T20:14:17,377 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc. 2024-11-18T20:14:17,377 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc. as already flushing 2024-11-18T20:14:17,377 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc. 2024-11-18T20:14:17,377 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 23d90021d1eda01703e7bdf7270f74fc, NAME => 'TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:14:17,377 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 23d90021d1eda01703e7bdf7270f74fc, NAME => 'TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:14:17,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 23d90021d1eda01703e7bdf7270f74fc, NAME => 'TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 23d90021d1eda01703e7bdf7270f74fc, NAME => 'TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:14:17,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-18T20:14:17,529 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35963 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-18T20:14:17,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc. 2024-11-18T20:14:17,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc. as already flushing 2024-11-18T20:14:17,530 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc. 2024-11-18T20:14:17,530 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 23d90021d1eda01703e7bdf7270f74fc, NAME => 'TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:14:17,530 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 23d90021d1eda01703e7bdf7270f74fc, NAME => 'TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:14:17,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 23d90021d1eda01703e7bdf7270f74fc, NAME => 'TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 23d90021d1eda01703e7bdf7270f74fc, NAME => 'TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:14:17,534 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:17,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51878 deadline: 1731960917533, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:17,535 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:17,535 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:17,535 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:14:17,546 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:17,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51894 deadline: 1731960917543, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:17,547 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:17,547 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:17,547 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:14:17,557 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:17,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51924 deadline: 1731960917553, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:17,558 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:17,558 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:17,558 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:14:17,683 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35963 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-18T20:14:17,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc. 2024-11-18T20:14:17,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc. as already flushing 2024-11-18T20:14:17,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc. 2024-11-18T20:14:17,683 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 23d90021d1eda01703e7bdf7270f74fc, NAME => 'TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:14:17,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 23d90021d1eda01703e7bdf7270f74fc, NAME => 'TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:14:17,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 23d90021d1eda01703e7bdf7270f74fc, NAME => 'TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 23d90021d1eda01703e7bdf7270f74fc, NAME => 'TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:14:17,715 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=235 (bloomFilter=true), to=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/.tmp/C/2df50040410a49a0ba45e6139fdd1a88 2024-11-18T20:14:17,720 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/.tmp/A/b4a146acc8b14703a2150a4f74d4d8b8 as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/A/b4a146acc8b14703a2150a4f74d4d8b8 2024-11-18T20:14:17,723 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/A/b4a146acc8b14703a2150a4f74d4d8b8, entries=200, sequenceid=235, filesize=14.2 K 2024-11-18T20:14:17,724 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/.tmp/B/f04cc9ecbf224d8791ec076ae5a8292a as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/B/f04cc9ecbf224d8791ec076ae5a8292a 2024-11-18T20:14:17,731 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/B/f04cc9ecbf224d8791ec076ae5a8292a, entries=150, sequenceid=235, filesize=11.9 K 2024-11-18T20:14:17,732 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/.tmp/C/2df50040410a49a0ba45e6139fdd1a88 as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/C/2df50040410a49a0ba45e6139fdd1a88 2024-11-18T20:14:17,737 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/C/2df50040410a49a0ba45e6139fdd1a88, entries=150, sequenceid=235, filesize=11.9 K 2024-11-18T20:14:17,738 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~167.72 KB/171750, heapSize ~440.16 KB/450720, currentSize=33.54 KB/34350 for 23d90021d1eda01703e7bdf7270f74fc in 857ms, sequenceid=235, compaction requested=true 2024-11-18T20:14:17,738 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 23d90021d1eda01703e7bdf7270f74fc: 2024-11-18T20:14:17,738 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 23d90021d1eda01703e7bdf7270f74fc:A, priority=-2147483648, current under compaction store size is 1 2024-11-18T20:14:17,738 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T20:14:17,738 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:14:17,738 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 23d90021d1eda01703e7bdf7270f74fc:B, priority=-2147483648, current under compaction store size is 2 2024-11-18T20:14:17,738 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:14:17,738 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 23d90021d1eda01703e7bdf7270f74fc:C, priority=-2147483648, current under compaction store size is 3 2024-11-18T20:14:17,738 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T20:14:17,738 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T20:14:17,739 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39287 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T20:14:17,740 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T20:14:17,740 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HStore(1541): 23d90021d1eda01703e7bdf7270f74fc/A is initiating minor compaction (all files) 2024-11-18T20:14:17,740 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HStore(1541): 23d90021d1eda01703e7bdf7270f74fc/B is initiating minor compaction (all files) 2024-11-18T20:14:17,740 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 23d90021d1eda01703e7bdf7270f74fc/A in TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc. 2024-11-18T20:14:17,740 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 23d90021d1eda01703e7bdf7270f74fc/B in TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc. 2024-11-18T20:14:17,740 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/A/eca3d1d04c9d47dc81c13429d7a59988, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/A/b21202671a9743f282ebd8a418b05a2f, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/A/b4a146acc8b14703a2150a4f74d4d8b8] into tmpdir=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/.tmp, totalSize=38.4 K 2024-11-18T20:14:17,740 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/B/ba25119d8ed745419b3bb21168a659c5, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/B/20c2c0e6b3c7463781ad28c4435ce89b, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/B/f04cc9ecbf224d8791ec076ae5a8292a] into tmpdir=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/.tmp, totalSize=36.0 K 2024-11-18T20:14:17,740 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] compactions.Compactor(225): Compacting eca3d1d04c9d47dc81c13429d7a59988, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1731960854523 2024-11-18T20:14:17,740 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.Compactor(225): Compacting ba25119d8ed745419b3bb21168a659c5, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1731960854523 2024-11-18T20:14:17,740 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] compactions.Compactor(225): Compacting b21202671a9743f282ebd8a418b05a2f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=205, earliestPutTs=1731960855180 2024-11-18T20:14:17,740 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.Compactor(225): Compacting 20c2c0e6b3c7463781ad28c4435ce89b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=205, earliestPutTs=1731960855180 2024-11-18T20:14:17,741 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.Compactor(225): Compacting f04cc9ecbf224d8791ec076ae5a8292a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1731960855944 2024-11-18T20:14:17,741 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] compactions.Compactor(225): Compacting b4a146acc8b14703a2150a4f74d4d8b8, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1731960855929 2024-11-18T20:14:17,749 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 23d90021d1eda01703e7bdf7270f74fc#B#compaction#432 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:14:17,749 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 23d90021d1eda01703e7bdf7270f74fc#A#compaction#431 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:14:17,749 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/.tmp/A/7ca07d43378f4ec2b8259dc478719d45 is 50, key is test_row_0/A:col10/1731960855944/Put/seqid=0 2024-11-18T20:14:17,749 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/.tmp/B/7fe6ad8f4bea4b379508d438f35ea4fa is 50, key is test_row_0/B:col10/1731960855944/Put/seqid=0 2024-11-18T20:14:17,758 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742332_1508 (size=12697) 2024-11-18T20:14:17,763 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742333_1509 (size=12697) 2024-11-18T20:14:17,835 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35963 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-18T20:14:17,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc. 2024-11-18T20:14:17,836 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2902): Flushing 23d90021d1eda01703e7bdf7270f74fc 3/3 column families, dataSize=33.54 KB heapSize=88.64 KB 2024-11-18T20:14:17,840 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/.tmp/A/4264dbcb302b4f73990ee2afdccfd4ed is 50, key is test_row_0/A:col10/1731960856901/Put/seqid=0 2024-11-18T20:14:17,844 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742334_1510 (size=9757) 2024-11-18T20:14:17,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-18T20:14:18,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(8855): Flush requested on 23d90021d1eda01703e7bdf7270f74fc 2024-11-18T20:14:18,054 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc. as already flushing 2024-11-18T20:14:18,149 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:18,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51878 deadline: 1731960918136, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:18,150 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:18,150 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:18,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51894 deadline: 1731960918136, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:18,150 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:18,150 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:14:18,150 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:18,150 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:18,150 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:18,150 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:14:18,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51862 deadline: 1731960918143, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:18,151 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to c0a89b2656d4,35963,1731960718223 for 'test_row_1' in 23d90021d1eda01703e7bdf7270f74fc of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2188 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-18T20:14:18,151 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:18,151 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:18,151 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:14:18,155 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:18,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51860 deadline: 1731960918148, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:18,155 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:18,155 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:18,156 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:14:18,160 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:18,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51924 deadline: 1731960918150, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:18,160 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:18,160 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:18,160 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:14:18,162 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/.tmp/B/7fe6ad8f4bea4b379508d438f35ea4fa as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/B/7fe6ad8f4bea4b379508d438f35ea4fa 2024-11-18T20:14:18,165 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 23d90021d1eda01703e7bdf7270f74fc/B of 23d90021d1eda01703e7bdf7270f74fc into 7fe6ad8f4bea4b379508d438f35ea4fa(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:14:18,165 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 23d90021d1eda01703e7bdf7270f74fc: 2024-11-18T20:14:18,165 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., storeName=23d90021d1eda01703e7bdf7270f74fc/B, priority=13, startTime=1731960857738; duration=0sec 2024-11-18T20:14:18,165 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T20:14:18,165 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 23d90021d1eda01703e7bdf7270f74fc:B 2024-11-18T20:14:18,165 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T20:14:18,166 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/.tmp/A/7ca07d43378f4ec2b8259dc478719d45 as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/A/7ca07d43378f4ec2b8259dc478719d45 2024-11-18T20:14:18,167 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T20:14:18,167 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HStore(1541): 23d90021d1eda01703e7bdf7270f74fc/C is initiating minor compaction (all files) 2024-11-18T20:14:18,167 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 23d90021d1eda01703e7bdf7270f74fc/C in TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc. 2024-11-18T20:14:18,168 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/C/b1e55172024f4005af135a2cd54f72d8, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/C/c833eb9b468546dbb7f321cdf01d8767, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/C/2df50040410a49a0ba45e6139fdd1a88] into tmpdir=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/.tmp, totalSize=36.0 K 2024-11-18T20:14:18,168 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.Compactor(225): Compacting b1e55172024f4005af135a2cd54f72d8, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1731960854523 2024-11-18T20:14:18,168 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.Compactor(225): Compacting c833eb9b468546dbb7f321cdf01d8767, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=205, earliestPutTs=1731960855180 2024-11-18T20:14:18,168 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.Compactor(225): Compacting 2df50040410a49a0ba45e6139fdd1a88, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1731960855944 2024-11-18T20:14:18,171 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 23d90021d1eda01703e7bdf7270f74fc/A of 23d90021d1eda01703e7bdf7270f74fc into 7ca07d43378f4ec2b8259dc478719d45(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:14:18,171 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 23d90021d1eda01703e7bdf7270f74fc: 2024-11-18T20:14:18,171 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., storeName=23d90021d1eda01703e7bdf7270f74fc/A, priority=13, startTime=1731960857738; duration=0sec 2024-11-18T20:14:18,172 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:14:18,172 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 23d90021d1eda01703e7bdf7270f74fc:A 2024-11-18T20:14:18,175 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 23d90021d1eda01703e7bdf7270f74fc#C#compaction#434 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:14:18,175 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/.tmp/C/f07ed15ec0d54cf2b03f54f59b562fc4 is 50, key is test_row_0/C:col10/1731960855944/Put/seqid=0 2024-11-18T20:14:18,178 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742335_1511 (size=12697) 2024-11-18T20:14:18,184 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/.tmp/C/f07ed15ec0d54cf2b03f54f59b562fc4 as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/C/f07ed15ec0d54cf2b03f54f59b562fc4 2024-11-18T20:14:18,188 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 23d90021d1eda01703e7bdf7270f74fc/C of 23d90021d1eda01703e7bdf7270f74fc into f07ed15ec0d54cf2b03f54f59b562fc4(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:14:18,188 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 23d90021d1eda01703e7bdf7270f74fc: 2024-11-18T20:14:18,188 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., storeName=23d90021d1eda01703e7bdf7270f74fc/C, priority=13, startTime=1731960857738; duration=0sec 2024-11-18T20:14:18,188 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:14:18,188 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 23d90021d1eda01703e7bdf7270f74fc:C 2024-11-18T20:14:18,245 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=243 (bloomFilter=true), to=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/.tmp/A/4264dbcb302b4f73990ee2afdccfd4ed 2024-11-18T20:14:18,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/.tmp/B/9e1a71695c5a4db9ba350390b278ffcc is 50, key is test_row_0/B:col10/1731960856901/Put/seqid=0 2024-11-18T20:14:18,256 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:18,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51878 deadline: 1731960918253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:18,257 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:18,257 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:18,257 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:18,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51894 deadline: 1731960918253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:18,257 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:14:18,257 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:18,257 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:18,257 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:14:18,262 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742336_1512 (size=9757) 2024-11-18T20:14:18,262 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=243 (bloomFilter=true), to=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/.tmp/B/9e1a71695c5a4db9ba350390b278ffcc 2024-11-18T20:14:18,267 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:18,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51860 deadline: 1731960918263, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:18,267 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:18,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51924 deadline: 1731960918264, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:18,267 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:18,267 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:18,268 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:14:18,268 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:18,268 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:18,268 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:14:18,272 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/.tmp/C/c6c190361b4a40908808f7757fbf4d7d is 50, key is test_row_0/C:col10/1731960856901/Put/seqid=0 2024-11-18T20:14:18,282 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742337_1513 (size=9757) 2024-11-18T20:14:18,283 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=243 (bloomFilter=true), to=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/.tmp/C/c6c190361b4a40908808f7757fbf4d7d 2024-11-18T20:14:18,288 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/.tmp/A/4264dbcb302b4f73990ee2afdccfd4ed as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/A/4264dbcb302b4f73990ee2afdccfd4ed 2024-11-18T20:14:18,291 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/A/4264dbcb302b4f73990ee2afdccfd4ed, entries=100, sequenceid=243, filesize=9.5 K 2024-11-18T20:14:18,291 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/.tmp/B/9e1a71695c5a4db9ba350390b278ffcc as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/B/9e1a71695c5a4db9ba350390b278ffcc 2024-11-18T20:14:18,294 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/B/9e1a71695c5a4db9ba350390b278ffcc, entries=100, sequenceid=243, filesize=9.5 K 2024-11-18T20:14:18,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/.tmp/C/c6c190361b4a40908808f7757fbf4d7d as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/C/c6c190361b4a40908808f7757fbf4d7d 2024-11-18T20:14:18,298 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/C/c6c190361b4a40908808f7757fbf4d7d, entries=100, sequenceid=243, filesize=9.5 K 2024-11-18T20:14:18,299 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=167.72 KB/171750 for 23d90021d1eda01703e7bdf7270f74fc in 464ms, sequenceid=243, compaction requested=false 2024-11-18T20:14:18,299 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2603): Flush status journal for 23d90021d1eda01703e7bdf7270f74fc: 2024-11-18T20:14:18,299 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc. 2024-11-18T20:14:18,299 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=132 2024-11-18T20:14:18,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.HMaster(4169): Remote procedure done, pid=132 2024-11-18T20:14:18,301 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=132, resume processing ppid=131 2024-11-18T20:14:18,301 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=132, ppid=131, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5330 sec 2024-11-18T20:14:18,302 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=131, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees in 1.5370 sec 2024-11-18T20:14:18,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(8855): Flush requested on 23d90021d1eda01703e7bdf7270f74fc 2024-11-18T20:14:18,466 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-18T20:14:18,466 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 23d90021d1eda01703e7bdf7270f74fc 3/3 column families, dataSize=174.43 KB heapSize=457.78 KB 2024-11-18T20:14:18,472 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/.tmp/A/527ecd9bc9544c6283c0b81917c2aee6 is 50, key is test_row_0/A:col10/1731960858465/Put/seqid=0 2024-11-18T20:14:18,475 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742338_1514 (size=17181) 2024-11-18T20:14:18,476 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=60.38 KB at sequenceid=276 (bloomFilter=true), to=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/.tmp/A/527ecd9bc9544c6283c0b81917c2aee6 2024-11-18T20:14:18,478 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:18,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51860 deadline: 1731960918473, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:18,479 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:18,479 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:18,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51924 deadline: 1731960918473, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:18,479 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:18,479 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:14:18,479 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:18,479 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:18,479 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:18,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51894 deadline: 1731960918476, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:18,479 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:14:18,479 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:18,480 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:18,480 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:14:18,484 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/.tmp/B/c5d22a92c4094ab2b149173de86be8d3 is 50, key is test_row_0/B:col10/1731960858465/Put/seqid=0 2024-11-18T20:14:18,485 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:18,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51878 deadline: 1731960918479, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:18,485 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:18,485 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:18,485 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:14:18,488 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742339_1515 (size=12301) 2024-11-18T20:14:18,587 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:18,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51894 deadline: 1731960918583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:18,587 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:18,587 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:18,587 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:14:18,597 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:18,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51878 deadline: 1731960918592, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:18,597 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:18,597 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:18,597 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:14:18,790 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:18,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51860 deadline: 1731960918789, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:18,790 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:18,790 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:18,790 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:14:18,796 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:18,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51924 deadline: 1731960918793, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:18,797 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:18,797 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:18,797 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:18,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51894 deadline: 1731960918793, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:18,797 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:14:18,797 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:18,797 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:18,797 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:14:18,806 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:18,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51878 deadline: 1731960918803, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:18,806 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:18,806 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:18,806 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:14:18,889 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=60.38 KB at sequenceid=276 (bloomFilter=true), to=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/.tmp/B/c5d22a92c4094ab2b149173de86be8d3 2024-11-18T20:14:18,899 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/.tmp/C/2be8cc2e5cf24b09a8e5455f9021590c is 50, key is test_row_0/C:col10/1731960858465/Put/seqid=0 2024-11-18T20:14:18,903 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742340_1516 (size=12301) 2024-11-18T20:14:18,905 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=60.38 KB at sequenceid=276 (bloomFilter=true), to=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/.tmp/C/2be8cc2e5cf24b09a8e5455f9021590c 2024-11-18T20:14:18,908 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/.tmp/A/527ecd9bc9544c6283c0b81917c2aee6 as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/A/527ecd9bc9544c6283c0b81917c2aee6 2024-11-18T20:14:18,911 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/A/527ecd9bc9544c6283c0b81917c2aee6, entries=250, sequenceid=276, filesize=16.8 K 2024-11-18T20:14:18,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-18T20:14:18,913 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-18T20:14:18,913 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-18T20:14:18,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] procedure2.ProcedureExecutor(1139): Stored pid=133, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees 2024-11-18T20:14:18,914 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/.tmp/B/c5d22a92c4094ab2b149173de86be8d3 as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/B/c5d22a92c4094ab2b149173de86be8d3 2024-11-18T20:14:18,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-18T20:14:18,915 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=133, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-18T20:14:18,915 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=133, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-18T20:14:18,915 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=134, ppid=133, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-18T20:14:18,919 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/B/c5d22a92c4094ab2b149173de86be8d3, entries=150, sequenceid=276, filesize=12.0 K 2024-11-18T20:14:18,919 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/.tmp/C/2be8cc2e5cf24b09a8e5455f9021590c as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/C/2be8cc2e5cf24b09a8e5455f9021590c 2024-11-18T20:14:18,922 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/C/2be8cc2e5cf24b09a8e5455f9021590c, entries=150, sequenceid=276, filesize=12.0 K 2024-11-18T20:14:18,922 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~181.14 KB/185490, heapSize ~475.31 KB/486720, currentSize=20.13 KB/20610 for 23d90021d1eda01703e7bdf7270f74fc in 456ms, sequenceid=276, compaction requested=true 2024-11-18T20:14:18,922 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 23d90021d1eda01703e7bdf7270f74fc: 2024-11-18T20:14:18,922 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 23d90021d1eda01703e7bdf7270f74fc:A, priority=-2147483648, current under compaction store size is 1 2024-11-18T20:14:18,922 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:14:18,922 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T20:14:18,922 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 23d90021d1eda01703e7bdf7270f74fc:B, priority=-2147483648, current under compaction store size is 2 2024-11-18T20:14:18,923 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:14:18,923 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T20:14:18,923 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 23d90021d1eda01703e7bdf7270f74fc:C, priority=-2147483648, current under compaction store size is 3 2024-11-18T20:14:18,923 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T20:14:18,924 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39635 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T20:14:18,924 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34755 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T20:14:18,924 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HStore(1541): 23d90021d1eda01703e7bdf7270f74fc/B is initiating minor compaction (all files) 2024-11-18T20:14:18,924 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HStore(1541): 23d90021d1eda01703e7bdf7270f74fc/A is initiating minor compaction (all files) 2024-11-18T20:14:18,924 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 23d90021d1eda01703e7bdf7270f74fc/A in TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc. 2024-11-18T20:14:18,924 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 23d90021d1eda01703e7bdf7270f74fc/B in TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc. 2024-11-18T20:14:18,924 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/A/7ca07d43378f4ec2b8259dc478719d45, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/A/4264dbcb302b4f73990ee2afdccfd4ed, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/A/527ecd9bc9544c6283c0b81917c2aee6] into tmpdir=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/.tmp, totalSize=38.7 K 2024-11-18T20:14:18,924 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/B/7fe6ad8f4bea4b379508d438f35ea4fa, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/B/9e1a71695c5a4db9ba350390b278ffcc, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/B/c5d22a92c4094ab2b149173de86be8d3] into tmpdir=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/.tmp, totalSize=33.9 K 2024-11-18T20:14:18,924 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.Compactor(225): Compacting 7fe6ad8f4bea4b379508d438f35ea4fa, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1731960855944 2024-11-18T20:14:18,925 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.Compactor(225): Compacting 9e1a71695c5a4db9ba350390b278ffcc, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=243, earliestPutTs=1731960856901 2024-11-18T20:14:18,925 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7ca07d43378f4ec2b8259dc478719d45, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1731960855944 2024-11-18T20:14:18,925 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4264dbcb302b4f73990ee2afdccfd4ed, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=243, earliestPutTs=1731960856901 2024-11-18T20:14:18,926 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] compactions.Compactor(225): Compacting 527ecd9bc9544c6283c0b81917c2aee6, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=276, earliestPutTs=1731960858107 2024-11-18T20:14:18,928 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.Compactor(225): Compacting c5d22a92c4094ab2b149173de86be8d3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=276, earliestPutTs=1731960858107 2024-11-18T20:14:18,933 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 23d90021d1eda01703e7bdf7270f74fc#A#compaction#440 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:14:18,934 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/.tmp/A/cb61591d6d5543efa1d3f004a444c70f is 50, key is test_row_0/A:col10/1731960858465/Put/seqid=0 2024-11-18T20:14:18,950 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 23d90021d1eda01703e7bdf7270f74fc#B#compaction#441 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:14:18,950 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/.tmp/B/d3403dff01bf4c90b12e27792941a4f5 is 50, key is test_row_0/B:col10/1731960858465/Put/seqid=0 2024-11-18T20:14:18,951 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742341_1517 (size=12949) 2024-11-18T20:14:18,956 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/.tmp/A/cb61591d6d5543efa1d3f004a444c70f as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/A/cb61591d6d5543efa1d3f004a444c70f 2024-11-18T20:14:18,960 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 23d90021d1eda01703e7bdf7270f74fc/A of 23d90021d1eda01703e7bdf7270f74fc into cb61591d6d5543efa1d3f004a444c70f(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:14:18,960 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 23d90021d1eda01703e7bdf7270f74fc: 2024-11-18T20:14:18,960 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., storeName=23d90021d1eda01703e7bdf7270f74fc/A, priority=13, startTime=1731960858922; duration=0sec 2024-11-18T20:14:18,960 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T20:14:18,960 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 23d90021d1eda01703e7bdf7270f74fc:A 2024-11-18T20:14:18,960 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T20:14:18,961 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34755 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T20:14:18,961 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HStore(1541): 23d90021d1eda01703e7bdf7270f74fc/C is initiating minor compaction (all files) 2024-11-18T20:14:18,961 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 23d90021d1eda01703e7bdf7270f74fc/C in TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc. 2024-11-18T20:14:18,961 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/C/f07ed15ec0d54cf2b03f54f59b562fc4, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/C/c6c190361b4a40908808f7757fbf4d7d, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/C/2be8cc2e5cf24b09a8e5455f9021590c] into tmpdir=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/.tmp, totalSize=33.9 K 2024-11-18T20:14:18,961 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] compactions.Compactor(225): Compacting f07ed15ec0d54cf2b03f54f59b562fc4, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1731960855944 2024-11-18T20:14:18,962 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] compactions.Compactor(225): Compacting c6c190361b4a40908808f7757fbf4d7d, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=243, earliestPutTs=1731960856901 2024-11-18T20:14:18,962 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2be8cc2e5cf24b09a8e5455f9021590c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=276, earliestPutTs=1731960858107 2024-11-18T20:14:18,968 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 23d90021d1eda01703e7bdf7270f74fc#C#compaction#442 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:14:18,968 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/.tmp/C/4273a1dead234073a9f9a921b2e591cb is 50, key is test_row_0/C:col10/1731960858465/Put/seqid=0 2024-11-18T20:14:18,970 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742342_1518 (size=12949) 2024-11-18T20:14:18,978 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742343_1519 (size=12949) 2024-11-18T20:14:18,983 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/.tmp/C/4273a1dead234073a9f9a921b2e591cb as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/C/4273a1dead234073a9f9a921b2e591cb 2024-11-18T20:14:18,986 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/.tmp/B/d3403dff01bf4c90b12e27792941a4f5 as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/B/d3403dff01bf4c90b12e27792941a4f5 2024-11-18T20:14:18,989 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 23d90021d1eda01703e7bdf7270f74fc/C of 23d90021d1eda01703e7bdf7270f74fc into 4273a1dead234073a9f9a921b2e591cb(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:14:18,989 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 23d90021d1eda01703e7bdf7270f74fc: 2024-11-18T20:14:18,989 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., storeName=23d90021d1eda01703e7bdf7270f74fc/C, priority=13, startTime=1731960858923; duration=0sec 2024-11-18T20:14:18,989 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:14:18,989 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 23d90021d1eda01703e7bdf7270f74fc:C 2024-11-18T20:14:18,990 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 23d90021d1eda01703e7bdf7270f74fc/B of 23d90021d1eda01703e7bdf7270f74fc into d3403dff01bf4c90b12e27792941a4f5(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:14:18,990 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 23d90021d1eda01703e7bdf7270f74fc: 2024-11-18T20:14:18,990 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., storeName=23d90021d1eda01703e7bdf7270f74fc/B, priority=13, startTime=1731960858922; duration=0sec 2024-11-18T20:14:18,991 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:14:18,991 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 23d90021d1eda01703e7bdf7270f74fc:B 2024-11-18T20:14:19,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-18T20:14:19,066 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35963 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-18T20:14:19,067 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc. 2024-11-18T20:14:19,067 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2902): Flushing 23d90021d1eda01703e7bdf7270f74fc 3/3 column families, dataSize=20.13 KB heapSize=53.48 KB 2024-11-18T20:14:19,071 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/.tmp/A/314f7a2e4ad344c5816bcb334990585b is 50, key is test_row_1/A:col10/1731960858474/Put/seqid=0 2024-11-18T20:14:19,075 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742344_1520 (size=9857) 2024-11-18T20:14:19,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(8855): Flush requested on 23d90021d1eda01703e7bdf7270f74fc 2024-11-18T20:14:19,113 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc. as already flushing 2024-11-18T20:14:19,174 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:19,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51878 deadline: 1731960919170, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:19,175 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:19,175 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:19,175 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:14:19,177 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:19,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51894 deadline: 1731960919171, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:19,177 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:19,177 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:19,177 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:14:19,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-18T20:14:19,286 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:19,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51878 deadline: 1731960919283, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:19,287 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:19,287 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:19,287 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:19,287 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:14:19,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51894 deadline: 1731960919283, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:19,287 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:19,287 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:19,287 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:14:19,307 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:19,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51860 deadline: 1731960919303, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:19,307 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:19,307 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:19,307 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:19,308 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:14:19,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51924 deadline: 1731960919303, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:19,308 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:19,308 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:19,308 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:14:19,476 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=285 (bloomFilter=true), to=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/.tmp/A/314f7a2e4ad344c5816bcb334990585b 2024-11-18T20:14:19,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/.tmp/B/11e335fd1bae486aa07c02ad707e8022 is 50, key is test_row_1/B:col10/1731960858474/Put/seqid=0 2024-11-18T20:14:19,484 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742345_1521 (size=9857) 2024-11-18T20:14:19,494 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:19,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51878 deadline: 1731960919493, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:19,495 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:19,495 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:19,495 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:19,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51894 deadline: 1731960919493, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:19,495 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:14:19,495 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:19,495 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:19,495 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:14:19,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-18T20:14:19,806 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:19,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51878 deadline: 1731960919803, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:19,806 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:19,806 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:19,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51894 deadline: 1731960919803, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:19,806 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:19,806 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:14:19,806 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:19,806 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:19,807 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:14:19,884 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=285 (bloomFilter=true), to=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/.tmp/B/11e335fd1bae486aa07c02ad707e8022 2024-11-18T20:14:19,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/.tmp/C/4e4de7e5bd99472e9841c2bdb3759f9d is 50, key is test_row_1/C:col10/1731960858474/Put/seqid=0 2024-11-18T20:14:19,893 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742346_1522 (size=9857) 2024-11-18T20:14:20,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-18T20:14:20,166 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:20,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51862 deadline: 1731960920163, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:20,167 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to c0a89b2656d4,35963,1731960718223 for 'test_row_1' in 23d90021d1eda01703e7bdf7270f74fc of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4204 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-18T20:14:20,167 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:20,167 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:20,167 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:14:20,294 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=285 (bloomFilter=true), to=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/.tmp/C/4e4de7e5bd99472e9841c2bdb3759f9d 2024-11-18T20:14:20,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/.tmp/A/314f7a2e4ad344c5816bcb334990585b as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/A/314f7a2e4ad344c5816bcb334990585b 2024-11-18T20:14:20,299 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/A/314f7a2e4ad344c5816bcb334990585b, entries=100, sequenceid=285, filesize=9.6 K 2024-11-18T20:14:20,300 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/.tmp/B/11e335fd1bae486aa07c02ad707e8022 as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/B/11e335fd1bae486aa07c02ad707e8022 2024-11-18T20:14:20,302 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/B/11e335fd1bae486aa07c02ad707e8022, entries=100, sequenceid=285, filesize=9.6 K 2024-11-18T20:14:20,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/.tmp/C/4e4de7e5bd99472e9841c2bdb3759f9d as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/C/4e4de7e5bd99472e9841c2bdb3759f9d 2024-11-18T20:14:20,305 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/C/4e4de7e5bd99472e9841c2bdb3759f9d, entries=100, sequenceid=285, filesize=9.6 K 2024-11-18T20:14:20,306 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(3140): Finished flush of dataSize ~20.13 KB/20610, heapSize ~53.44 KB/54720, currentSize=181.14 KB/185490 for 23d90021d1eda01703e7bdf7270f74fc in 1239ms, sequenceid=285, compaction requested=false 2024-11-18T20:14:20,306 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2603): Flush status journal for 23d90021d1eda01703e7bdf7270f74fc: 2024-11-18T20:14:20,306 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc. 2024-11-18T20:14:20,306 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=134 2024-11-18T20:14:20,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.HMaster(4169): Remote procedure done, pid=134 2024-11-18T20:14:20,309 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=134, resume processing ppid=133 2024-11-18T20:14:20,309 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=134, ppid=133, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3920 sec 2024-11-18T20:14:20,310 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=133, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees in 1.3960 sec 2024-11-18T20:14:20,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(8855): Flush requested on 23d90021d1eda01703e7bdf7270f74fc 2024-11-18T20:14:20,317 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-18T20:14:20,318 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 23d90021d1eda01703e7bdf7270f74fc 3/3 column families, dataSize=187.85 KB heapSize=492.94 KB 2024-11-18T20:14:20,323 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/.tmp/A/072bcff7b0ea4b9385fec058d55a3d6a is 50, key is test_row_0/A:col10/1731960860316/Put/seqid=0 2024-11-18T20:14:20,324 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:20,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51894 deadline: 1731960920319, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:20,325 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:20,325 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:20,325 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:20,325 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:14:20,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51878 deadline: 1731960920321, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:20,325 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:20,325 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:20,325 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:14:20,327 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742347_1523 (size=17181) 2024-11-18T20:14:20,327 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=64.85 KB at sequenceid=317 (bloomFilter=true), to=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/.tmp/A/072bcff7b0ea4b9385fec058d55a3d6a 2024-11-18T20:14:20,327 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:20,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51860 deadline: 1731960920323, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:20,328 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:20,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51924 deadline: 1731960920323, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:20,328 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to c0a89b2656d4,35963,1731960718223 for 'test_row_1' in 23d90021d1eda01703e7bdf7270f74fc of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2180 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-18T20:14:20,328 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:20,328 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:20,328 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:14:20,328 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to c0a89b2656d4,35963,1731960718223 for 'test_row_1' in 23d90021d1eda01703e7bdf7270f74fc of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2178 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-18T20:14:20,328 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:20,328 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:20,329 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:14:20,334 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/.tmp/B/5734456e867d486697ec69ca3d131044 is 50, key is test_row_0/B:col10/1731960860316/Put/seqid=0 2024-11-18T20:14:20,337 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742348_1524 (size=12301) 2024-11-18T20:14:20,338 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=64.85 KB at sequenceid=317 (bloomFilter=true), to=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/.tmp/B/5734456e867d486697ec69ca3d131044 2024-11-18T20:14:20,343 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/.tmp/C/4fb476bab21e4462a958d9bd9b563251 is 50, key is test_row_0/C:col10/1731960860316/Put/seqid=0 2024-11-18T20:14:20,345 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742349_1525 (size=12301) 2024-11-18T20:14:20,437 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:20,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51894 deadline: 1731960920433, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:20,437 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:20,437 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:20,437 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:20,438 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:14:20,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51878 deadline: 1731960920433, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:20,438 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:20,438 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:20,438 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:14:20,647 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:20,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51894 deadline: 1731960920643, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:20,648 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:20,647 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:20,648 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:20,648 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:14:20,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51878 deadline: 1731960920643, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:20,648 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:20,648 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:20,648 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:14:20,746 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=64.85 KB at sequenceid=317 (bloomFilter=true), to=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/.tmp/C/4fb476bab21e4462a958d9bd9b563251 2024-11-18T20:14:20,749 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/.tmp/A/072bcff7b0ea4b9385fec058d55a3d6a as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/A/072bcff7b0ea4b9385fec058d55a3d6a 2024-11-18T20:14:20,752 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/A/072bcff7b0ea4b9385fec058d55a3d6a, entries=250, sequenceid=317, filesize=16.8 K 2024-11-18T20:14:20,753 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/.tmp/B/5734456e867d486697ec69ca3d131044 as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/B/5734456e867d486697ec69ca3d131044 2024-11-18T20:14:20,755 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/B/5734456e867d486697ec69ca3d131044, entries=150, sequenceid=317, filesize=12.0 K 2024-11-18T20:14:20,756 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/.tmp/C/4fb476bab21e4462a958d9bd9b563251 as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/C/4fb476bab21e4462a958d9bd9b563251 2024-11-18T20:14:20,760 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/C/4fb476bab21e4462a958d9bd9b563251, entries=150, sequenceid=317, filesize=12.0 K 2024-11-18T20:14:20,760 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~194.56 KB/199230, heapSize ~510.47 KB/522720, currentSize=6.71 KB/6870 for 23d90021d1eda01703e7bdf7270f74fc in 443ms, sequenceid=317, compaction requested=true 2024-11-18T20:14:20,761 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 23d90021d1eda01703e7bdf7270f74fc: 2024-11-18T20:14:20,761 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 23d90021d1eda01703e7bdf7270f74fc:A, priority=-2147483648, current under compaction store size is 1 2024-11-18T20:14:20,761 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:14:20,761 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 23d90021d1eda01703e7bdf7270f74fc:B, priority=-2147483648, current under compaction store size is 2 2024-11-18T20:14:20,761 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T20:14:20,761 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:14:20,761 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 23d90021d1eda01703e7bdf7270f74fc:C, priority=-2147483648, current under compaction store size is 3 2024-11-18T20:14:20,761 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T20:14:20,761 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T20:14:20,761 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39987 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T20:14:20,761 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35107 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T20:14:20,761 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HStore(1541): 23d90021d1eda01703e7bdf7270f74fc/B is initiating minor compaction (all files) 2024-11-18T20:14:20,761 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HStore(1541): 23d90021d1eda01703e7bdf7270f74fc/A is initiating minor compaction (all files) 2024-11-18T20:14:20,761 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 23d90021d1eda01703e7bdf7270f74fc/A in TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc. 2024-11-18T20:14:20,761 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 23d90021d1eda01703e7bdf7270f74fc/B in TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc. 2024-11-18T20:14:20,762 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/B/d3403dff01bf4c90b12e27792941a4f5, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/B/11e335fd1bae486aa07c02ad707e8022, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/B/5734456e867d486697ec69ca3d131044] into tmpdir=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/.tmp, totalSize=34.3 K 2024-11-18T20:14:20,762 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/A/cb61591d6d5543efa1d3f004a444c70f, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/A/314f7a2e4ad344c5816bcb334990585b, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/A/072bcff7b0ea4b9385fec058d55a3d6a] into tmpdir=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/.tmp, totalSize=39.0 K 2024-11-18T20:14:20,762 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.Compactor(225): Compacting d3403dff01bf4c90b12e27792941a4f5, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=276, earliestPutTs=1731960858107 2024-11-18T20:14:20,762 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] compactions.Compactor(225): Compacting cb61591d6d5543efa1d3f004a444c70f, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=276, earliestPutTs=1731960858107 2024-11-18T20:14:20,762 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.Compactor(225): Compacting 11e335fd1bae486aa07c02ad707e8022, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=285, earliestPutTs=1731960858474 2024-11-18T20:14:20,762 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] compactions.Compactor(225): Compacting 314f7a2e4ad344c5816bcb334990585b, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=285, earliestPutTs=1731960858474 2024-11-18T20:14:20,762 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.Compactor(225): Compacting 5734456e867d486697ec69ca3d131044, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=317, earliestPutTs=1731960859159 2024-11-18T20:14:20,762 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] compactions.Compactor(225): Compacting 072bcff7b0ea4b9385fec058d55a3d6a, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=317, earliestPutTs=1731960859159 2024-11-18T20:14:20,773 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 23d90021d1eda01703e7bdf7270f74fc#B#compaction#449 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:14:20,773 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/.tmp/B/24d2d39e80434108877eecd7d9b554ec is 50, key is test_row_0/B:col10/1731960860316/Put/seqid=0 2024-11-18T20:14:20,774 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 23d90021d1eda01703e7bdf7270f74fc#A#compaction#450 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:14:20,774 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/.tmp/A/948983a4aec144a187ef3168e340f4a6 is 50, key is test_row_0/A:col10/1731960860316/Put/seqid=0 2024-11-18T20:14:20,778 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742350_1526 (size=13051) 2024-11-18T20:14:20,780 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742351_1527 (size=13051) 2024-11-18T20:14:20,782 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/.tmp/B/24d2d39e80434108877eecd7d9b554ec as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/B/24d2d39e80434108877eecd7d9b554ec 2024-11-18T20:14:20,788 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 23d90021d1eda01703e7bdf7270f74fc/B of 23d90021d1eda01703e7bdf7270f74fc into 24d2d39e80434108877eecd7d9b554ec(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:14:20,788 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 23d90021d1eda01703e7bdf7270f74fc: 2024-11-18T20:14:20,788 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., storeName=23d90021d1eda01703e7bdf7270f74fc/B, priority=13, startTime=1731960860761; duration=0sec 2024-11-18T20:14:20,788 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T20:14:20,788 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 23d90021d1eda01703e7bdf7270f74fc:B 2024-11-18T20:14:20,788 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T20:14:20,789 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35107 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T20:14:20,789 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HStore(1541): 23d90021d1eda01703e7bdf7270f74fc/C is initiating minor compaction (all files) 2024-11-18T20:14:20,789 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 23d90021d1eda01703e7bdf7270f74fc/C in TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc. 2024-11-18T20:14:20,789 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/C/4273a1dead234073a9f9a921b2e591cb, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/C/4e4de7e5bd99472e9841c2bdb3759f9d, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/C/4fb476bab21e4462a958d9bd9b563251] into tmpdir=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/.tmp, totalSize=34.3 K 2024-11-18T20:14:20,790 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.Compactor(225): Compacting 4273a1dead234073a9f9a921b2e591cb, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=276, earliestPutTs=1731960858107 2024-11-18T20:14:20,790 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.Compactor(225): Compacting 4e4de7e5bd99472e9841c2bdb3759f9d, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=285, earliestPutTs=1731960858474 2024-11-18T20:14:20,791 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.Compactor(225): Compacting 4fb476bab21e4462a958d9bd9b563251, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=317, earliestPutTs=1731960859159 2024-11-18T20:14:20,799 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 23d90021d1eda01703e7bdf7270f74fc#C#compaction#451 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:14:20,799 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/.tmp/C/9df14f10b8134882b9ed3fa9f1976ca8 is 50, key is test_row_0/C:col10/1731960860316/Put/seqid=0 2024-11-18T20:14:20,806 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742352_1528 (size=13051) 2024-11-18T20:14:20,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(8855): Flush requested on 23d90021d1eda01703e7bdf7270f74fc 2024-11-18T20:14:20,969 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-18T20:14:20,970 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 23d90021d1eda01703e7bdf7270f74fc 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-18T20:14:20,977 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/.tmp/A/ea5ba5ce27444144a56e5ed79a073dcf is 50, key is test_row_0/A:col10/1731960860964/Put/seqid=0 2024-11-18T20:14:20,980 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742353_1529 (size=17181) 2024-11-18T20:14:21,048 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:21,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51878 deadline: 1731960921045, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:21,048 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:21,048 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:21,048 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:14:21,053 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:21,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51894 deadline: 1731960921047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:21,054 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:21,054 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:21,054 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:14:21,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-18T20:14:21,063 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-18T20:14:21,063 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-18T20:14:21,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] procedure2.ProcedureExecutor(1139): Stored pid=135, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees 2024-11-18T20:14:21,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-18T20:14:21,065 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=135, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-18T20:14:21,065 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=135, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-18T20:14:21,065 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=136, ppid=135, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-18T20:14:21,157 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:21,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51878 deadline: 1731960921153, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:21,157 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:21,157 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:21,157 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:14:21,166 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:21,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51894 deadline: 1731960921163, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:21,166 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:21,166 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:21,166 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:14:21,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-18T20:14:21,183 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/.tmp/A/948983a4aec144a187ef3168e340f4a6 as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/A/948983a4aec144a187ef3168e340f4a6 2024-11-18T20:14:21,187 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 23d90021d1eda01703e7bdf7270f74fc/A of 23d90021d1eda01703e7bdf7270f74fc into 948983a4aec144a187ef3168e340f4a6(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:14:21,187 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 23d90021d1eda01703e7bdf7270f74fc: 2024-11-18T20:14:21,187 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., storeName=23d90021d1eda01703e7bdf7270f74fc/A, priority=13, startTime=1731960860761; duration=0sec 2024-11-18T20:14:21,187 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:14:21,187 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 23d90021d1eda01703e7bdf7270f74fc:A 2024-11-18T20:14:21,210 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/.tmp/C/9df14f10b8134882b9ed3fa9f1976ca8 as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/C/9df14f10b8134882b9ed3fa9f1976ca8 2024-11-18T20:14:21,214 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 23d90021d1eda01703e7bdf7270f74fc/C of 23d90021d1eda01703e7bdf7270f74fc into 9df14f10b8134882b9ed3fa9f1976ca8(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:14:21,214 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 23d90021d1eda01703e7bdf7270f74fc: 2024-11-18T20:14:21,214 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., storeName=23d90021d1eda01703e7bdf7270f74fc/C, priority=13, startTime=1731960860761; duration=0sec 2024-11-18T20:14:21,214 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:14:21,214 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 23d90021d1eda01703e7bdf7270f74fc:C 2024-11-18T20:14:21,217 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35963 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-18T20:14:21,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc. 2024-11-18T20:14:21,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc. as already flushing 2024-11-18T20:14:21,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc. 2024-11-18T20:14:21,218 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => 23d90021d1eda01703e7bdf7270f74fc, NAME => 'TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:14:21,218 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => 23d90021d1eda01703e7bdf7270f74fc, NAME => 'TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:14:21,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 23d90021d1eda01703e7bdf7270f74fc, NAME => 'TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 23d90021d1eda01703e7bdf7270f74fc, NAME => 'TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:14:21,368 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:21,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51878 deadline: 1731960921363, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:21,368 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:21,368 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:21,368 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:14:21,370 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35963 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-18T20:14:21,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc. 2024-11-18T20:14:21,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc. as already flushing 2024-11-18T20:14:21,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc. 2024-11-18T20:14:21,370 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => 23d90021d1eda01703e7bdf7270f74fc, NAME => 'TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:14:21,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => 23d90021d1eda01703e7bdf7270f74fc, NAME => 'TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:14:21,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 23d90021d1eda01703e7bdf7270f74fc, NAME => 'TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 23d90021d1eda01703e7bdf7270f74fc, NAME => 'TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:14:21,379 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:21,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51894 deadline: 1731960921373, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:21,379 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:21,379 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:21,379 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:14:21,381 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=329 (bloomFilter=true), to=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/.tmp/A/ea5ba5ce27444144a56e5ed79a073dcf 2024-11-18T20:14:21,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-18T20:14:21,386 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/.tmp/B/167b31563e074e498fbd5e79b99f6954 is 50, key is test_row_0/B:col10/1731960860964/Put/seqid=0 2024-11-18T20:14:21,389 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742354_1530 (size=12301) 2024-11-18T20:14:21,390 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=329 (bloomFilter=true), to=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/.tmp/B/167b31563e074e498fbd5e79b99f6954 2024-11-18T20:14:21,395 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/.tmp/C/79279deee29e45c381a0a64b8afa5ec0 is 50, key is test_row_0/C:col10/1731960860964/Put/seqid=0 2024-11-18T20:14:21,404 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742355_1531 (size=12301) 2024-11-18T20:14:21,522 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35963 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-18T20:14:21,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc. 2024-11-18T20:14:21,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc. as already flushing 2024-11-18T20:14:21,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc. 2024-11-18T20:14:21,522 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => 23d90021d1eda01703e7bdf7270f74fc, NAME => 'TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:14:21,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => 23d90021d1eda01703e7bdf7270f74fc, NAME => 'TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:14:21,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 23d90021d1eda01703e7bdf7270f74fc, NAME => 'TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 23d90021d1eda01703e7bdf7270f74fc, NAME => 'TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:14:21,675 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35963 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-18T20:14:21,675 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc. 2024-11-18T20:14:21,675 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc. as already flushing 2024-11-18T20:14:21,675 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc. 2024-11-18T20:14:21,675 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => 23d90021d1eda01703e7bdf7270f74fc, NAME => 'TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:14:21,675 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => 23d90021d1eda01703e7bdf7270f74fc, NAME => 'TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:14:21,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 23d90021d1eda01703e7bdf7270f74fc, NAME => 'TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 23d90021d1eda01703e7bdf7270f74fc, NAME => 'TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:14:21,680 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:21,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51878 deadline: 1731960921672, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:21,681 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:21,681 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:21,681 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:14:21,688 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:21,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51894 deadline: 1731960921683, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:21,688 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:21,688 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:21,688 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:14:21,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-18T20:14:21,805 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=329 (bloomFilter=true), to=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/.tmp/C/79279deee29e45c381a0a64b8afa5ec0 2024-11-18T20:14:21,808 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/.tmp/A/ea5ba5ce27444144a56e5ed79a073dcf as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/A/ea5ba5ce27444144a56e5ed79a073dcf 2024-11-18T20:14:21,810 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/A/ea5ba5ce27444144a56e5ed79a073dcf, entries=250, sequenceid=329, filesize=16.8 K 2024-11-18T20:14:21,810 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/.tmp/B/167b31563e074e498fbd5e79b99f6954 as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/B/167b31563e074e498fbd5e79b99f6954 2024-11-18T20:14:21,813 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/B/167b31563e074e498fbd5e79b99f6954, entries=150, sequenceid=329, filesize=12.0 K 2024-11-18T20:14:21,813 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/.tmp/C/79279deee29e45c381a0a64b8afa5ec0 as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/C/79279deee29e45c381a0a64b8afa5ec0 2024-11-18T20:14:21,816 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/C/79279deee29e45c381a0a64b8afa5ec0, entries=150, sequenceid=329, filesize=12.0 K 2024-11-18T20:14:21,817 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 23d90021d1eda01703e7bdf7270f74fc in 848ms, sequenceid=329, compaction requested=false 2024-11-18T20:14:21,817 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 23d90021d1eda01703e7bdf7270f74fc: 2024-11-18T20:14:21,827 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35963 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-18T20:14:21,828 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc. 2024-11-18T20:14:21,828 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2902): Flushing 23d90021d1eda01703e7bdf7270f74fc 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-18T20:14:21,831 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/.tmp/A/c37937f85b9b454b8df620dd1fc2bb4f is 50, key is test_row_0/A:col10/1731960861044/Put/seqid=0 2024-11-18T20:14:21,836 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742356_1532 (size=12301) 2024-11-18T20:14:22,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(8855): Flush requested on 23d90021d1eda01703e7bdf7270f74fc 2024-11-18T20:14:22,197 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc. as already flushing 2024-11-18T20:14:22,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-18T20:14:22,223 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:22,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51894 deadline: 1731960922222, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:22,223 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:22,223 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:22,223 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:14:22,234 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:22,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51878 deadline: 1731960922223, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:22,234 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:22,234 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:22,234 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:14:22,236 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=356 (bloomFilter=true), to=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/.tmp/A/c37937f85b9b454b8df620dd1fc2bb4f 2024-11-18T20:14:22,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/.tmp/B/3a9791a87f974a959254d83efffbba24 is 50, key is test_row_0/B:col10/1731960861044/Put/seqid=0 2024-11-18T20:14:22,245 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742357_1533 (size=12301) 2024-11-18T20:14:22,246 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=356 (bloomFilter=true), to=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/.tmp/B/3a9791a87f974a959254d83efffbba24 2024-11-18T20:14:22,251 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/.tmp/C/cd713709f7b348b78cf489b5313a50c3 is 50, key is test_row_0/C:col10/1731960861044/Put/seqid=0 2024-11-18T20:14:22,258 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742358_1534 (size=12301) 2024-11-18T20:14:22,338 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:22,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51894 deadline: 1731960922333, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:22,339 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:22,339 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:22,339 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:14:22,346 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:22,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51924 deadline: 1731960922343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:22,346 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:22,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51878 deadline: 1731960922343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:22,347 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:22,347 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:22,347 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:14:22,346 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to c0a89b2656d4,35963,1731960718223 for 'test_row_1' in 23d90021d1eda01703e7bdf7270f74fc of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4197 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-18T20:14:22,347 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:22,347 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:22,347 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:14:22,358 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:22,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51860 deadline: 1731960922353, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:22,358 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to c0a89b2656d4,35963,1731960718223 for 'test_row_1' in 23d90021d1eda01703e7bdf7270f74fc of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4211 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-18T20:14:22,359 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:22,359 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:22,359 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:14:22,549 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:22,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51894 deadline: 1731960922543, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:22,550 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:22,550 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:22,550 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:14:22,557 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:22,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51878 deadline: 1731960922553, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:22,558 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:22,558 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:22,558 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:14:22,656 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=356 (bloomFilter=true), to=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/.tmp/C/cd713709f7b348b78cf489b5313a50c3 2024-11-18T20:14:22,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/.tmp/A/c37937f85b9b454b8df620dd1fc2bb4f as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/A/c37937f85b9b454b8df620dd1fc2bb4f 2024-11-18T20:14:22,661 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/A/c37937f85b9b454b8df620dd1fc2bb4f, entries=150, sequenceid=356, filesize=12.0 K 2024-11-18T20:14:22,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/.tmp/B/3a9791a87f974a959254d83efffbba24 as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/B/3a9791a87f974a959254d83efffbba24 2024-11-18T20:14:22,664 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/B/3a9791a87f974a959254d83efffbba24, entries=150, sequenceid=356, filesize=12.0 K 2024-11-18T20:14:22,665 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/.tmp/C/cd713709f7b348b78cf489b5313a50c3 as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/C/cd713709f7b348b78cf489b5313a50c3 2024-11-18T20:14:22,668 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/C/cd713709f7b348b78cf489b5313a50c3, entries=150, sequenceid=356, filesize=12.0 K 2024-11-18T20:14:22,669 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=60.38 KB/61830 for 23d90021d1eda01703e7bdf7270f74fc in 841ms, sequenceid=356, compaction requested=true 2024-11-18T20:14:22,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2603): Flush status journal for 23d90021d1eda01703e7bdf7270f74fc: 2024-11-18T20:14:22,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc. 2024-11-18T20:14:22,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=136 2024-11-18T20:14:22,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.HMaster(4169): Remote procedure done, pid=136 2024-11-18T20:14:22,671 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=136, resume processing ppid=135 2024-11-18T20:14:22,671 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=136, ppid=135, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6050 sec 2024-11-18T20:14:22,672 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=135, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees in 1.6080 sec 2024-11-18T20:14:22,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(8855): Flush requested on 23d90021d1eda01703e7bdf7270f74fc 2024-11-18T20:14:22,856 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-18T20:14:22,856 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 23d90021d1eda01703e7bdf7270f74fc 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-18T20:14:22,859 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/.tmp/A/c29b9784d9464777bdb1e2aa57e27495 is 50, key is test_row_0/A:col10/1731960862855/Put/seqid=0 2024-11-18T20:14:22,863 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742359_1535 (size=14741) 2024-11-18T20:14:22,864 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=369 (bloomFilter=true), to=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/.tmp/A/c29b9784d9464777bdb1e2aa57e27495 2024-11-18T20:14:22,868 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/.tmp/B/9f5e9e1fdbbd4af2b6530f36483166c3 is 50, key is test_row_0/B:col10/1731960862855/Put/seqid=0 2024-11-18T20:14:22,880 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742360_1536 (size=12301) 2024-11-18T20:14:22,880 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=369 (bloomFilter=true), to=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/.tmp/B/9f5e9e1fdbbd4af2b6530f36483166c3 2024-11-18T20:14:22,887 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/.tmp/C/0eb3fc69063f4efb8562407794df502a is 50, key is test_row_0/C:col10/1731960862855/Put/seqid=0 2024-11-18T20:14:22,899 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742361_1537 (size=12301) 2024-11-18T20:14:22,900 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=369 (bloomFilter=true), to=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/.tmp/C/0eb3fc69063f4efb8562407794df502a 2024-11-18T20:14:22,902 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/.tmp/A/c29b9784d9464777bdb1e2aa57e27495 as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/A/c29b9784d9464777bdb1e2aa57e27495 2024-11-18T20:14:22,907 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/A/c29b9784d9464777bdb1e2aa57e27495, entries=200, sequenceid=369, filesize=14.4 K 2024-11-18T20:14:22,907 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/.tmp/B/9f5e9e1fdbbd4af2b6530f36483166c3 as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/B/9f5e9e1fdbbd4af2b6530f36483166c3 2024-11-18T20:14:22,910 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/B/9f5e9e1fdbbd4af2b6530f36483166c3, entries=150, sequenceid=369, filesize=12.0 K 2024-11-18T20:14:22,911 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/.tmp/C/0eb3fc69063f4efb8562407794df502a as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/C/0eb3fc69063f4efb8562407794df502a 2024-11-18T20:14:22,913 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/C/0eb3fc69063f4efb8562407794df502a, entries=150, sequenceid=369, filesize=12.0 K 2024-11-18T20:14:22,919 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=53.67 KB/54960 for 23d90021d1eda01703e7bdf7270f74fc in 63ms, sequenceid=369, compaction requested=true 2024-11-18T20:14:22,919 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 23d90021d1eda01703e7bdf7270f74fc: 2024-11-18T20:14:22,919 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 23d90021d1eda01703e7bdf7270f74fc:A, priority=-2147483648, current under compaction store size is 1 2024-11-18T20:14:22,919 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-18T20:14:22,919 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:14:22,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(8855): Flush requested on 23d90021d1eda01703e7bdf7270f74fc 2024-11-18T20:14:22,919 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 23d90021d1eda01703e7bdf7270f74fc:B, priority=-2147483648, current under compaction store size is 2 2024-11-18T20:14:22,919 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:14:22,919 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-18T20:14:22,919 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 23d90021d1eda01703e7bdf7270f74fc:C, priority=-2147483648, current under compaction store size is 3 2024-11-18T20:14:22,920 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T20:14:22,920 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-18T20:14:22,920 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 23d90021d1eda01703e7bdf7270f74fc 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-18T20:14:22,921 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49954 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-18T20:14:22,921 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HStore(1541): 23d90021d1eda01703e7bdf7270f74fc/B is initiating minor compaction (all files) 2024-11-18T20:14:22,921 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 57274 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-18T20:14:22,921 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HStore(1541): 23d90021d1eda01703e7bdf7270f74fc/A is initiating minor compaction (all files) 2024-11-18T20:14:22,921 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 23d90021d1eda01703e7bdf7270f74fc/B in TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc. 2024-11-18T20:14:22,921 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 23d90021d1eda01703e7bdf7270f74fc/A in TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc. 2024-11-18T20:14:22,921 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/B/24d2d39e80434108877eecd7d9b554ec, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/B/167b31563e074e498fbd5e79b99f6954, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/B/3a9791a87f974a959254d83efffbba24, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/B/9f5e9e1fdbbd4af2b6530f36483166c3] into tmpdir=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/.tmp, totalSize=48.8 K 2024-11-18T20:14:22,921 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/A/948983a4aec144a187ef3168e340f4a6, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/A/ea5ba5ce27444144a56e5ed79a073dcf, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/A/c37937f85b9b454b8df620dd1fc2bb4f, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/A/c29b9784d9464777bdb1e2aa57e27495] into tmpdir=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/.tmp, totalSize=55.9 K 2024-11-18T20:14:22,922 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.Compactor(225): Compacting 24d2d39e80434108877eecd7d9b554ec, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=317, earliestPutTs=1731960859159 2024-11-18T20:14:22,922 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] compactions.Compactor(225): Compacting 948983a4aec144a187ef3168e340f4a6, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=317, earliestPutTs=1731960859159 2024-11-18T20:14:22,922 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] compactions.Compactor(225): Compacting ea5ba5ce27444144a56e5ed79a073dcf, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=329, earliestPutTs=1731960860959 2024-11-18T20:14:22,922 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.Compactor(225): Compacting 167b31563e074e498fbd5e79b99f6954, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=329, earliestPutTs=1731960860959 2024-11-18T20:14:22,922 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] compactions.Compactor(225): Compacting c37937f85b9b454b8df620dd1fc2bb4f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=356, earliestPutTs=1731960860997 2024-11-18T20:14:22,922 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.Compactor(225): Compacting 3a9791a87f974a959254d83efffbba24, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=356, earliestPutTs=1731960860997 2024-11-18T20:14:22,922 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] compactions.Compactor(225): Compacting c29b9784d9464777bdb1e2aa57e27495, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=369, earliestPutTs=1731960862212 2024-11-18T20:14:22,922 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.Compactor(225): Compacting 9f5e9e1fdbbd4af2b6530f36483166c3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=369, earliestPutTs=1731960862212 2024-11-18T20:14:22,923 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/.tmp/A/684367518a0a49e8bca6eb6083817683 is 50, key is test_row_0/A:col10/1731960862896/Put/seqid=0 2024-11-18T20:14:22,926 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742362_1538 (size=17181) 2024-11-18T20:14:22,927 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=382 (bloomFilter=true), to=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/.tmp/A/684367518a0a49e8bca6eb6083817683 2024-11-18T20:14:22,929 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 23d90021d1eda01703e7bdf7270f74fc#A#compaction#462 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:14:22,930 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/.tmp/A/ab6ed2a3421a47f2b8bee325c2a5c392 is 50, key is test_row_0/A:col10/1731960862855/Put/seqid=0 2024-11-18T20:14:22,930 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 23d90021d1eda01703e7bdf7270f74fc#B#compaction#463 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:14:22,930 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/.tmp/B/d798bf9a6f2148bc9e2e4b29f4e841fe is 50, key is test_row_0/B:col10/1731960862855/Put/seqid=0 2024-11-18T20:14:22,932 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/.tmp/B/8922649520834958b53a7bbbcd7fb257 is 50, key is test_row_0/B:col10/1731960862896/Put/seqid=0 2024-11-18T20:14:22,934 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742363_1539 (size=13187) 2024-11-18T20:14:22,936 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742364_1540 (size=13187) 2024-11-18T20:14:22,942 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742365_1541 (size=12301) 2024-11-18T20:14:23,029 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:23,029 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:23,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51894 deadline: 1731960923021, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:23,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51878 deadline: 1731960923017, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:23,030 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:23,030 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:23,030 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:23,030 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:23,030 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:14:23,030 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:14:23,139 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:23,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51878 deadline: 1731960923132, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:23,140 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:23,140 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:23,140 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:23,140 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:14:23,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51894 deadline: 1731960923133, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:23,140 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:23,140 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:23,140 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:14:23,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-18T20:14:23,214 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-18T20:14:23,214 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-18T20:14:23,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] procedure2.ProcedureExecutor(1139): Stored pid=137, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=137, table=TestAcidGuarantees 2024-11-18T20:14:23,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-18T20:14:23,215 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=137, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=137, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-18T20:14:23,216 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=137, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=137, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-18T20:14:23,216 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=138, ppid=137, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-18T20:14:23,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-18T20:14:23,337 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/.tmp/A/ab6ed2a3421a47f2b8bee325c2a5c392 as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/A/ab6ed2a3421a47f2b8bee325c2a5c392 2024-11-18T20:14:23,339 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/.tmp/B/d798bf9a6f2148bc9e2e4b29f4e841fe as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/B/d798bf9a6f2148bc9e2e4b29f4e841fe 2024-11-18T20:14:23,340 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 23d90021d1eda01703e7bdf7270f74fc/A of 23d90021d1eda01703e7bdf7270f74fc into ab6ed2a3421a47f2b8bee325c2a5c392(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:14:23,340 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 23d90021d1eda01703e7bdf7270f74fc: 2024-11-18T20:14:23,340 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., storeName=23d90021d1eda01703e7bdf7270f74fc/A, priority=12, startTime=1731960862919; duration=0sec 2024-11-18T20:14:23,340 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T20:14:23,340 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 23d90021d1eda01703e7bdf7270f74fc:A 2024-11-18T20:14:23,340 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-18T20:14:23,342 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49954 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-18T20:14:23,342 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HStore(1541): 23d90021d1eda01703e7bdf7270f74fc/C is initiating minor compaction (all files) 2024-11-18T20:14:23,342 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 23d90021d1eda01703e7bdf7270f74fc/C in TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc. 2024-11-18T20:14:23,342 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/C/9df14f10b8134882b9ed3fa9f1976ca8, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/C/79279deee29e45c381a0a64b8afa5ec0, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/C/cd713709f7b348b78cf489b5313a50c3, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/C/0eb3fc69063f4efb8562407794df502a] into tmpdir=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/.tmp, totalSize=48.8 K 2024-11-18T20:14:23,342 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9df14f10b8134882b9ed3fa9f1976ca8, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=317, earliestPutTs=1731960859159 2024-11-18T20:14:23,343 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] compactions.Compactor(225): Compacting 79279deee29e45c381a0a64b8afa5ec0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=329, earliestPutTs=1731960860959 2024-11-18T20:14:23,343 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=382 (bloomFilter=true), to=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/.tmp/B/8922649520834958b53a7bbbcd7fb257 2024-11-18T20:14:23,343 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] compactions.Compactor(225): Compacting cd713709f7b348b78cf489b5313a50c3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=356, earliestPutTs=1731960860997 2024-11-18T20:14:23,343 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0eb3fc69063f4efb8562407794df502a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=369, earliestPutTs=1731960862212 2024-11-18T20:14:23,344 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 23d90021d1eda01703e7bdf7270f74fc/B of 23d90021d1eda01703e7bdf7270f74fc into d798bf9a6f2148bc9e2e4b29f4e841fe(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:14:23,344 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 23d90021d1eda01703e7bdf7270f74fc: 2024-11-18T20:14:23,344 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., storeName=23d90021d1eda01703e7bdf7270f74fc/B, priority=12, startTime=1731960862919; duration=0sec 2024-11-18T20:14:23,344 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:14:23,344 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 23d90021d1eda01703e7bdf7270f74fc:B 2024-11-18T20:14:23,349 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/.tmp/C/8008d6970185499784364a1ae6ce1e69 is 50, key is test_row_0/C:col10/1731960862896/Put/seqid=0 2024-11-18T20:14:23,351 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 23d90021d1eda01703e7bdf7270f74fc#C#compaction#466 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:14:23,351 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:23,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51894 deadline: 1731960923343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:23,352 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:23,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51878 deadline: 1731960923343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:23,352 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/.tmp/C/49db5d906b88433ba35319adc4b7bcb5 is 50, key is test_row_0/C:col10/1731960862855/Put/seqid=0 2024-11-18T20:14:23,352 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:23,352 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:23,352 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:14:23,353 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:23,353 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:23,353 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:14:23,366 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742367_1543 (size=13187) 2024-11-18T20:14:23,367 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35963 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-11-18T20:14:23,367 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc. 2024-11-18T20:14:23,367 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc. as already flushing 2024-11-18T20:14:23,368 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc. 2024-11-18T20:14:23,368 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] handler.RSProcedureHandler(58): pid=138 java.io.IOException: Unable to complete flush {ENCODED => 23d90021d1eda01703e7bdf7270f74fc, NAME => 'TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:14:23,368 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=138 java.io.IOException: Unable to complete flush {ENCODED => 23d90021d1eda01703e7bdf7270f74fc, NAME => 'TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:14:23,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.HMaster(4177): Remote procedure failed, pid=138 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 23d90021d1eda01703e7bdf7270f74fc, NAME => 'TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 23d90021d1eda01703e7bdf7270f74fc, NAME => 'TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:14:23,377 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742366_1542 (size=12301) 2024-11-18T20:14:23,520 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35963 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-11-18T20:14:23,520 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc. 2024-11-18T20:14:23,520 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc. as already flushing 2024-11-18T20:14:23,520 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc. 2024-11-18T20:14:23,520 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] handler.RSProcedureHandler(58): pid=138 java.io.IOException: Unable to complete flush {ENCODED => 23d90021d1eda01703e7bdf7270f74fc, NAME => 'TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:14:23,520 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=138 java.io.IOException: Unable to complete flush {ENCODED => 23d90021d1eda01703e7bdf7270f74fc, NAME => 'TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:14:23,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.HMaster(4177): Remote procedure failed, pid=138 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 23d90021d1eda01703e7bdf7270f74fc, NAME => 'TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 23d90021d1eda01703e7bdf7270f74fc, NAME => 'TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:14:23,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-18T20:14:23,666 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:23,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51878 deadline: 1731960923663, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:23,667 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:23,667 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:23,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51894 deadline: 1731960923663, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:23,667 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:23,667 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:14:23,667 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:23,667 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:23,667 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:14:23,673 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35963 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-11-18T20:14:23,673 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc. 2024-11-18T20:14:23,673 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc. as already flushing 2024-11-18T20:14:23,674 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc. 2024-11-18T20:14:23,674 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] handler.RSProcedureHandler(58): pid=138 java.io.IOException: Unable to complete flush {ENCODED => 23d90021d1eda01703e7bdf7270f74fc, NAME => 'TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:14:23,674 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=138 java.io.IOException: Unable to complete flush {ENCODED => 23d90021d1eda01703e7bdf7270f74fc, NAME => 'TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:14:23,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.HMaster(4177): Remote procedure failed, pid=138 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 23d90021d1eda01703e7bdf7270f74fc, NAME => 'TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 23d90021d1eda01703e7bdf7270f74fc, NAME => 'TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:14:23,770 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/.tmp/C/49db5d906b88433ba35319adc4b7bcb5 as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/C/49db5d906b88433ba35319adc4b7bcb5 2024-11-18T20:14:23,774 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 23d90021d1eda01703e7bdf7270f74fc/C of 23d90021d1eda01703e7bdf7270f74fc into 49db5d906b88433ba35319adc4b7bcb5(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:14:23,774 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 23d90021d1eda01703e7bdf7270f74fc: 2024-11-18T20:14:23,774 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., storeName=23d90021d1eda01703e7bdf7270f74fc/C, priority=12, startTime=1731960862919; duration=0sec 2024-11-18T20:14:23,774 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:14:23,774 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 23d90021d1eda01703e7bdf7270f74fc:C 2024-11-18T20:14:23,777 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=382 (bloomFilter=true), to=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/.tmp/C/8008d6970185499784364a1ae6ce1e69 2024-11-18T20:14:23,780 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/.tmp/A/684367518a0a49e8bca6eb6083817683 as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/A/684367518a0a49e8bca6eb6083817683 2024-11-18T20:14:23,783 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/A/684367518a0a49e8bca6eb6083817683, entries=250, sequenceid=382, filesize=16.8 K 2024-11-18T20:14:23,784 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/.tmp/B/8922649520834958b53a7bbbcd7fb257 as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/B/8922649520834958b53a7bbbcd7fb257 2024-11-18T20:14:23,792 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/B/8922649520834958b53a7bbbcd7fb257, entries=150, sequenceid=382, filesize=12.0 K 2024-11-18T20:14:23,793 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/.tmp/C/8008d6970185499784364a1ae6ce1e69 as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/C/8008d6970185499784364a1ae6ce1e69 2024-11-18T20:14:23,796 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/C/8008d6970185499784364a1ae6ce1e69, entries=150, sequenceid=382, filesize=12.0 K 2024-11-18T20:14:23,796 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for 23d90021d1eda01703e7bdf7270f74fc in 876ms, sequenceid=382, compaction requested=false 2024-11-18T20:14:23,796 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 23d90021d1eda01703e7bdf7270f74fc: 2024-11-18T20:14:23,826 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35963 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-11-18T20:14:23,826 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc. 2024-11-18T20:14:23,826 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2902): Flushing 23d90021d1eda01703e7bdf7270f74fc 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-18T20:14:23,830 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/.tmp/A/a357e04cd5c94668a56d3e496d92f4aa is 50, key is test_row_0/A:col10/1731960862956/Put/seqid=0 2024-11-18T20:14:23,834 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742368_1544 (size=12301) 2024-11-18T20:14:23,834 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=408 (bloomFilter=true), to=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/.tmp/A/a357e04cd5c94668a56d3e496d92f4aa 2024-11-18T20:14:23,840 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/.tmp/B/5de4f933d11049d788fea5a2edeb5285 is 50, key is test_row_0/B:col10/1731960862956/Put/seqid=0 2024-11-18T20:14:23,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-18T20:14:23,845 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742369_1545 (size=12301) 2024-11-18T20:14:23,846 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=408 (bloomFilter=true), to=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/.tmp/B/5de4f933d11049d788fea5a2edeb5285 2024-11-18T20:14:23,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/.tmp/C/5ed42a638aa1426a973974e899dc69b0 is 50, key is test_row_0/C:col10/1731960862956/Put/seqid=0 2024-11-18T20:14:23,874 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742370_1546 (size=12301) 2024-11-18T20:14:23,879 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=408 (bloomFilter=true), to=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/.tmp/C/5ed42a638aa1426a973974e899dc69b0 2024-11-18T20:14:23,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/.tmp/A/a357e04cd5c94668a56d3e496d92f4aa as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/A/a357e04cd5c94668a56d3e496d92f4aa 2024-11-18T20:14:23,892 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/A/a357e04cd5c94668a56d3e496d92f4aa, entries=150, sequenceid=408, filesize=12.0 K 2024-11-18T20:14:23,893 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/.tmp/B/5de4f933d11049d788fea5a2edeb5285 as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/B/5de4f933d11049d788fea5a2edeb5285 2024-11-18T20:14:23,898 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/B/5de4f933d11049d788fea5a2edeb5285, entries=150, sequenceid=408, filesize=12.0 K 2024-11-18T20:14:23,899 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/.tmp/C/5ed42a638aa1426a973974e899dc69b0 as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/C/5ed42a638aa1426a973974e899dc69b0 2024-11-18T20:14:23,905 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/C/5ed42a638aa1426a973974e899dc69b0, entries=150, sequenceid=408, filesize=12.0 K 2024-11-18T20:14:23,907 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=0 B/0 for 23d90021d1eda01703e7bdf7270f74fc in 80ms, sequenceid=408, compaction requested=true 2024-11-18T20:14:23,907 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2603): Flush status journal for 23d90021d1eda01703e7bdf7270f74fc: 2024-11-18T20:14:23,907 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc. 2024-11-18T20:14:23,907 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=138 2024-11-18T20:14:23,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.HMaster(4169): Remote procedure done, pid=138 2024-11-18T20:14:23,909 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=138, resume processing ppid=137 2024-11-18T20:14:23,910 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=138, ppid=137, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 692 msec 2024-11-18T20:14:23,912 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=137, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=137, table=TestAcidGuarantees in 696 msec 2024-11-18T20:14:24,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(8855): Flush requested on 23d90021d1eda01703e7bdf7270f74fc 2024-11-18T20:14:24,185 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-18T20:14:24,186 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 23d90021d1eda01703e7bdf7270f74fc 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-18T20:14:24,189 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/.tmp/A/051fb4c1e17b4254a130fc59629ffe48 is 50, key is test_row_0/A:col10/1731960864184/Put/seqid=0 2024-11-18T20:14:24,192 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742371_1547 (size=14737) 2024-11-18T20:14:24,240 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:24,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51894 deadline: 1731960924237, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:24,241 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:24,241 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:24,241 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:14:24,246 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:24,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51862 deadline: 1731960924238, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:24,246 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:24,246 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:24,246 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:14:24,247 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:24,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51878 deadline: 1731960924239, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:24,247 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:24,247 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:24,248 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:14:24,346 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:24,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51894 deadline: 1731960924343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:24,347 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:24,347 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:24,347 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:14:24,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-18T20:14:24,353 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-18T20:14:24,353 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-18T20:14:24,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] procedure2.ProcedureExecutor(1139): Stored pid=139, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=139, table=TestAcidGuarantees 2024-11-18T20:14:24,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-18T20:14:24,355 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=139, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=139, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-18T20:14:24,356 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=139, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=139, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-18T20:14:24,356 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=140, ppid=139, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-18T20:14:24,360 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:24,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51862 deadline: 1731960924353, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:24,360 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:24,360 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:24,360 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:24,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51878 deadline: 1731960924353, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:24,360 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:14:24,361 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:24,361 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:24,361 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:14:24,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-18T20:14:24,507 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35963 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=140 2024-11-18T20:14:24,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc. 2024-11-18T20:14:24,508 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc. as already flushing 2024-11-18T20:14:24,508 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc. 2024-11-18T20:14:24,508 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] handler.RSProcedureHandler(58): pid=140 java.io.IOException: Unable to complete flush {ENCODED => 23d90021d1eda01703e7bdf7270f74fc, NAME => 'TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:14:24,508 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=140 java.io.IOException: Unable to complete flush {ENCODED => 23d90021d1eda01703e7bdf7270f74fc, NAME => 'TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:14:24,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.HMaster(4177): Remote procedure failed, pid=140 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 23d90021d1eda01703e7bdf7270f74fc, NAME => 'TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 23d90021d1eda01703e7bdf7270f74fc, NAME => 'TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:14:24,556 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:24,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51894 deadline: 1731960924553, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:24,556 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:24,556 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:24,556 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:14:24,569 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:24,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51862 deadline: 1731960924563, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:24,570 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:24,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51878 deadline: 1731960924563, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:24,571 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:24,571 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:24,571 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:14:24,581 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:24,581 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:24,581 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:14:24,593 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=419 (bloomFilter=true), to=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/.tmp/A/051fb4c1e17b4254a130fc59629ffe48 2024-11-18T20:14:24,600 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/.tmp/B/d7115417c82e4375924463412f23eb99 is 50, key is test_row_0/B:col10/1731960864184/Put/seqid=0 2024-11-18T20:14:24,603 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742372_1548 (size=9857) 2024-11-18T20:14:24,660 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35963 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=140 2024-11-18T20:14:24,660 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc. 2024-11-18T20:14:24,660 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc. as already flushing 2024-11-18T20:14:24,660 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc. 2024-11-18T20:14:24,660 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] handler.RSProcedureHandler(58): pid=140 java.io.IOException: Unable to complete flush {ENCODED => 23d90021d1eda01703e7bdf7270f74fc, NAME => 'TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:14:24,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=140 java.io.IOException: Unable to complete flush {ENCODED => 23d90021d1eda01703e7bdf7270f74fc, NAME => 'TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:14:24,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.HMaster(4177): Remote procedure failed, pid=140 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 23d90021d1eda01703e7bdf7270f74fc, NAME => 'TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 23d90021d1eda01703e7bdf7270f74fc, NAME => 'TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:14:24,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-18T20:14:24,813 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35963 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=140 2024-11-18T20:14:24,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc. 2024-11-18T20:14:24,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc. as already flushing 2024-11-18T20:14:24,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc. 2024-11-18T20:14:24,813 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] handler.RSProcedureHandler(58): pid=140 java.io.IOException: Unable to complete flush {ENCODED => 23d90021d1eda01703e7bdf7270f74fc, NAME => 'TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:14:24,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=140 java.io.IOException: Unable to complete flush {ENCODED => 23d90021d1eda01703e7bdf7270f74fc, NAME => 'TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:14:24,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.HMaster(4177): Remote procedure failed, pid=140 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 23d90021d1eda01703e7bdf7270f74fc, NAME => 'TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 23d90021d1eda01703e7bdf7270f74fc, NAME => 'TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:14:24,866 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:24,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51894 deadline: 1731960924862, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:24,866 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:24,866 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:24,866 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:14:24,887 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:24,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51878 deadline: 1731960924883, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:24,887 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:24,887 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:24,887 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:24,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51862 deadline: 1731960924883, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:24,887 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:14:24,888 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:24,888 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:24,888 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:14:24,966 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35963 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=140 2024-11-18T20:14:24,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc. 2024-11-18T20:14:24,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc. as already flushing 2024-11-18T20:14:24,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc. 2024-11-18T20:14:24,966 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] handler.RSProcedureHandler(58): pid=140 java.io.IOException: Unable to complete flush {ENCODED => 23d90021d1eda01703e7bdf7270f74fc, NAME => 'TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:14:24,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=140 java.io.IOException: Unable to complete flush {ENCODED => 23d90021d1eda01703e7bdf7270f74fc, NAME => 'TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:14:24,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.HMaster(4177): Remote procedure failed, pid=140 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 23d90021d1eda01703e7bdf7270f74fc, NAME => 'TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 23d90021d1eda01703e7bdf7270f74fc, NAME => 'TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:14:24,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-18T20:14:25,003 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=419 (bloomFilter=true), to=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/.tmp/B/d7115417c82e4375924463412f23eb99 2024-11-18T20:14:25,009 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/.tmp/C/9f18f3f38b304a058486899db8dbed2a is 50, key is test_row_0/C:col10/1731960864184/Put/seqid=0 2024-11-18T20:14:25,011 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742373_1549 (size=9857) 2024-11-18T20:14:25,023 INFO [Thread-2064 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2064. 2024-11-18T20:14:25,024 DEBUG [Thread-2064 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-18T20:14:25,024 DEBUG [Thread-2064 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:14:25,024 DEBUG [Thread-2064 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:14:25,024 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T20:14:25,026 INFO [Thread-2068 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2068. 2024-11-18T20:14:25,026 DEBUG [Thread-2068 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-18T20:14:25,026 DEBUG [Thread-2068 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:14:25,026 DEBUG [Thread-2068 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:14:25,026 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T20:14:25,027 INFO [Thread-2070 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2070. 2024-11-18T20:14:25,027 DEBUG [Thread-2070 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-18T20:14:25,027 DEBUG [Thread-2070 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:14:25,027 DEBUG [Thread-2070 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:14:25,027 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T20:14:25,033 INFO [Thread-2066 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2066. 2024-11-18T20:14:25,033 DEBUG [Thread-2066 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-18T20:14:25,033 DEBUG [Thread-2066 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:14:25,033 DEBUG [Thread-2066 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:14:25,033 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T20:14:25,036 INFO [Thread-2072 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2072. 2024-11-18T20:14:25,036 DEBUG [Thread-2072 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-18T20:14:25,036 DEBUG [Thread-2072 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:14:25,036 DEBUG [Thread-2072 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:14:25,036 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T20:14:25,119 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35963 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=140 2024-11-18T20:14:25,119 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc. 2024-11-18T20:14:25,119 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc. as already flushing 2024-11-18T20:14:25,119 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc. 2024-11-18T20:14:25,119 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] handler.RSProcedureHandler(58): pid=140 java.io.IOException: Unable to complete flush {ENCODED => 23d90021d1eda01703e7bdf7270f74fc, NAME => 'TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:14:25,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=140 java.io.IOException: Unable to complete flush {ENCODED => 23d90021d1eda01703e7bdf7270f74fc, NAME => 'TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:14:25,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.HMaster(4177): Remote procedure failed, pid=140 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 23d90021d1eda01703e7bdf7270f74fc, NAME => 'TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 23d90021d1eda01703e7bdf7270f74fc, NAME => 'TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:14:25,273 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35963 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=140 2024-11-18T20:14:25,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc. 2024-11-18T20:14:25,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc. as already flushing 2024-11-18T20:14:25,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc. 2024-11-18T20:14:25,274 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] handler.RSProcedureHandler(58): pid=140 java.io.IOException: Unable to complete flush {ENCODED => 23d90021d1eda01703e7bdf7270f74fc, NAME => 'TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:14:25,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=140 java.io.IOException: Unable to complete flush {ENCODED => 23d90021d1eda01703e7bdf7270f74fc, NAME => 'TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:14:25,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.HMaster(4177): Remote procedure failed, pid=140 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 23d90021d1eda01703e7bdf7270f74fc, NAME => 'TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 23d90021d1eda01703e7bdf7270f74fc, NAME => 'TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:14:25,374 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:25,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51894 deadline: 1731960925374, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:25,375 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:25,375 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:25,375 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:14:25,394 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:25,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51878 deadline: 1731960925393, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:25,394 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:25,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51862 deadline: 1731960925394, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:25,395 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:25,395 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:25,395 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 , the old value is region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:25,395 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:14:25,395 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:25,395 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., hostname=c0a89b2656d4,35963,1731960718223, seqNum=2 because the exception is null or not the one we care about 2024-11-18T20:14:25,413 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=419 (bloomFilter=true), to=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/.tmp/C/9f18f3f38b304a058486899db8dbed2a 2024-11-18T20:14:25,422 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/.tmp/A/051fb4c1e17b4254a130fc59629ffe48 as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/A/051fb4c1e17b4254a130fc59629ffe48 2024-11-18T20:14:25,427 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/A/051fb4c1e17b4254a130fc59629ffe48, entries=200, sequenceid=419, filesize=14.4 K 2024-11-18T20:14:25,427 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35963 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=140 2024-11-18T20:14:25,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc. 2024-11-18T20:14:25,427 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/.tmp/B/d7115417c82e4375924463412f23eb99 as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/B/d7115417c82e4375924463412f23eb99 2024-11-18T20:14:25,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc. as already flushing 2024-11-18T20:14:25,428 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc. 2024-11-18T20:14:25,428 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] handler.RSProcedureHandler(58): pid=140 java.io.IOException: Unable to complete flush {ENCODED => 23d90021d1eda01703e7bdf7270f74fc, NAME => 'TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:14:25,428 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=140 java.io.IOException: Unable to complete flush {ENCODED => 23d90021d1eda01703e7bdf7270f74fc, NAME => 'TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:14:25,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.HMaster(4177): Remote procedure failed, pid=140 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 23d90021d1eda01703e7bdf7270f74fc, NAME => 'TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 23d90021d1eda01703e7bdf7270f74fc, NAME => 'TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:14:25,432 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/B/d7115417c82e4375924463412f23eb99, entries=100, sequenceid=419, filesize=9.6 K 2024-11-18T20:14:25,432 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/.tmp/C/9f18f3f38b304a058486899db8dbed2a as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/C/9f18f3f38b304a058486899db8dbed2a 2024-11-18T20:14:25,436 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/C/9f18f3f38b304a058486899db8dbed2a, entries=100, sequenceid=419, filesize=9.6 K 2024-11-18T20:14:25,437 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 23d90021d1eda01703e7bdf7270f74fc in 1251ms, sequenceid=419, compaction requested=true 2024-11-18T20:14:25,437 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 23d90021d1eda01703e7bdf7270f74fc: 2024-11-18T20:14:25,437 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 23d90021d1eda01703e7bdf7270f74fc:A, priority=-2147483648, current under compaction store size is 1 2024-11-18T20:14:25,437 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:14:25,437 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-18T20:14:25,437 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 23d90021d1eda01703e7bdf7270f74fc:B, priority=-2147483648, current under compaction store size is 2 2024-11-18T20:14:25,438 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:14:25,438 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-18T20:14:25,438 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 23d90021d1eda01703e7bdf7270f74fc:C, priority=-2147483648, current under compaction store size is 3 2024-11-18T20:14:25,438 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T20:14:25,439 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 47646 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-18T20:14:25,439 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 57406 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-18T20:14:25,439 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HStore(1541): 23d90021d1eda01703e7bdf7270f74fc/B is initiating minor compaction (all files) 2024-11-18T20:14:25,439 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HStore(1541): 23d90021d1eda01703e7bdf7270f74fc/A is initiating minor compaction (all files) 2024-11-18T20:14:25,439 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 23d90021d1eda01703e7bdf7270f74fc/B in TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc. 2024-11-18T20:14:25,439 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 23d90021d1eda01703e7bdf7270f74fc/A in TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc. 2024-11-18T20:14:25,439 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/B/d798bf9a6f2148bc9e2e4b29f4e841fe, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/B/8922649520834958b53a7bbbcd7fb257, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/B/5de4f933d11049d788fea5a2edeb5285, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/B/d7115417c82e4375924463412f23eb99] into tmpdir=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/.tmp, totalSize=46.5 K 2024-11-18T20:14:25,439 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/A/ab6ed2a3421a47f2b8bee325c2a5c392, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/A/684367518a0a49e8bca6eb6083817683, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/A/a357e04cd5c94668a56d3e496d92f4aa, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/A/051fb4c1e17b4254a130fc59629ffe48] into tmpdir=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/.tmp, totalSize=56.1 K 2024-11-18T20:14:25,439 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] compactions.Compactor(225): Compacting ab6ed2a3421a47f2b8bee325c2a5c392, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=369, earliestPutTs=1731960862212 2024-11-18T20:14:25,439 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.Compactor(225): Compacting d798bf9a6f2148bc9e2e4b29f4e841fe, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=369, earliestPutTs=1731960862212 2024-11-18T20:14:25,439 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.Compactor(225): Compacting 8922649520834958b53a7bbbcd7fb257, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=382, earliestPutTs=1731960862890 2024-11-18T20:14:25,439 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] compactions.Compactor(225): Compacting 684367518a0a49e8bca6eb6083817683, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=382, earliestPutTs=1731960862881 2024-11-18T20:14:25,440 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.Compactor(225): Compacting 5de4f933d11049d788fea5a2edeb5285, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=408, earliestPutTs=1731960862956 2024-11-18T20:14:25,440 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] compactions.Compactor(225): Compacting a357e04cd5c94668a56d3e496d92f4aa, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=408, earliestPutTs=1731960862956 2024-11-18T20:14:25,440 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.Compactor(225): Compacting d7115417c82e4375924463412f23eb99, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=419, earliestPutTs=1731960864181 2024-11-18T20:14:25,440 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] compactions.Compactor(225): Compacting 051fb4c1e17b4254a130fc59629ffe48, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=419, earliestPutTs=1731960864176 2024-11-18T20:14:25,445 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 23d90021d1eda01703e7bdf7270f74fc#B#compaction#474 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:14:25,445 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 23d90021d1eda01703e7bdf7270f74fc#A#compaction#473 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:14:25,446 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/.tmp/B/a366f8eda60840f2873deb2a8f979083 is 50, key is test_row_0/B:col10/1731960864184/Put/seqid=0 2024-11-18T20:14:25,446 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/.tmp/A/5d9ed599fb2c4d30a726d81af79c3c20 is 50, key is test_row_0/A:col10/1731960864184/Put/seqid=0 2024-11-18T20:14:25,448 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742375_1551 (size=13323) 2024-11-18T20:14:25,448 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742374_1550 (size=13323) 2024-11-18T20:14:25,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-18T20:14:25,581 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35963 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=140 2024-11-18T20:14:25,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc. 2024-11-18T20:14:25,582 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(2902): Flushing 23d90021d1eda01703e7bdf7270f74fc 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-18T20:14:25,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/.tmp/A/4cb386e694d14b3cae02cfc805ffb9f9 is 50, key is test_row_0/A:col10/1731960864237/Put/seqid=0 2024-11-18T20:14:25,590 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742376_1552 (size=12301) 2024-11-18T20:14:25,852 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/.tmp/B/a366f8eda60840f2873deb2a8f979083 as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/B/a366f8eda60840f2873deb2a8f979083 2024-11-18T20:14:25,852 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/.tmp/A/5d9ed599fb2c4d30a726d81af79c3c20 as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/A/5d9ed599fb2c4d30a726d81af79c3c20 2024-11-18T20:14:25,857 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 23d90021d1eda01703e7bdf7270f74fc/A of 23d90021d1eda01703e7bdf7270f74fc into 5d9ed599fb2c4d30a726d81af79c3c20(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:14:25,857 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 23d90021d1eda01703e7bdf7270f74fc/B of 23d90021d1eda01703e7bdf7270f74fc into a366f8eda60840f2873deb2a8f979083(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:14:25,857 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 23d90021d1eda01703e7bdf7270f74fc: 2024-11-18T20:14:25,857 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 23d90021d1eda01703e7bdf7270f74fc: 2024-11-18T20:14:25,857 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., storeName=23d90021d1eda01703e7bdf7270f74fc/A, priority=12, startTime=1731960865437; duration=0sec 2024-11-18T20:14:25,857 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., storeName=23d90021d1eda01703e7bdf7270f74fc/B, priority=12, startTime=1731960865437; duration=0sec 2024-11-18T20:14:25,857 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T20:14:25,857 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 23d90021d1eda01703e7bdf7270f74fc:B 2024-11-18T20:14:25,857 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T20:14:25,857 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 23d90021d1eda01703e7bdf7270f74fc:A 2024-11-18T20:14:25,857 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-18T20:14:25,859 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 47646 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-18T20:14:25,859 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HStore(1541): 23d90021d1eda01703e7bdf7270f74fc/C is initiating minor compaction (all files) 2024-11-18T20:14:25,859 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 23d90021d1eda01703e7bdf7270f74fc/C in TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc. 2024-11-18T20:14:25,859 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/C/49db5d906b88433ba35319adc4b7bcb5, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/C/8008d6970185499784364a1ae6ce1e69, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/C/5ed42a638aa1426a973974e899dc69b0, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/C/9f18f3f38b304a058486899db8dbed2a] into tmpdir=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/.tmp, totalSize=46.5 K 2024-11-18T20:14:25,859 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.Compactor(225): Compacting 49db5d906b88433ba35319adc4b7bcb5, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=369, earliestPutTs=1731960862212 2024-11-18T20:14:25,860 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.Compactor(225): Compacting 8008d6970185499784364a1ae6ce1e69, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=382, earliestPutTs=1731960862890 2024-11-18T20:14:25,860 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.Compactor(225): Compacting 5ed42a638aa1426a973974e899dc69b0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=408, earliestPutTs=1731960862956 2024-11-18T20:14:25,860 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.Compactor(225): Compacting 9f18f3f38b304a058486899db8dbed2a, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=419, earliestPutTs=1731960864181 2024-11-18T20:14:25,868 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 23d90021d1eda01703e7bdf7270f74fc#C#compaction#476 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:14:25,868 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/.tmp/C/24ee05f43a0544d5925ec842af646cd2 is 50, key is test_row_0/C:col10/1731960864184/Put/seqid=0 2024-11-18T20:14:25,870 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742377_1553 (size=13323) 2024-11-18T20:14:25,991 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=444 (bloomFilter=true), to=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/.tmp/A/4cb386e694d14b3cae02cfc805ffb9f9 2024-11-18T20:14:26,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/.tmp/B/ded17ca865fa42bea9593eb489fc8656 is 50, key is test_row_0/B:col10/1731960864237/Put/seqid=0 2024-11-18T20:14:26,010 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742378_1554 (size=12301) 2024-11-18T20:14:26,274 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/.tmp/C/24ee05f43a0544d5925ec842af646cd2 as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/C/24ee05f43a0544d5925ec842af646cd2 2024-11-18T20:14:26,277 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 23d90021d1eda01703e7bdf7270f74fc/C of 23d90021d1eda01703e7bdf7270f74fc into 24ee05f43a0544d5925ec842af646cd2(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:14:26,277 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 23d90021d1eda01703e7bdf7270f74fc: 2024-11-18T20:14:26,277 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., storeName=23d90021d1eda01703e7bdf7270f74fc/C, priority=12, startTime=1731960865438; duration=0sec 2024-11-18T20:14:26,277 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:14:26,277 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 23d90021d1eda01703e7bdf7270f74fc:C 2024-11-18T20:14:26,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(8855): Flush requested on 23d90021d1eda01703e7bdf7270f74fc 2024-11-18T20:14:26,354 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc. as already flushing 2024-11-18T20:14:26,354 INFO [Thread-2057 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2057. 2024-11-18T20:14:26,354 DEBUG [Thread-2057 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-18T20:14:26,354 DEBUG [Thread-2057 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:14:26,354 DEBUG [Thread-2057 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:14:26,354 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T20:14:26,385 INFO [Thread-2053 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2053. 2024-11-18T20:14:26,385 DEBUG [Thread-2053 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-18T20:14:26,385 DEBUG [Thread-2053 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:14:26,385 DEBUG [Thread-2053 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:14:26,385 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T20:14:26,395 INFO [Thread-2061 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2061. 2024-11-18T20:14:26,395 DEBUG [Thread-2061 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-18T20:14:26,395 DEBUG [Thread-2061 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:14:26,395 DEBUG [Thread-2061 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:14:26,395 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T20:14:26,404 INFO [Thread-2055 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2055. 2024-11-18T20:14:26,405 DEBUG [Thread-2055 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-18T20:14:26,405 DEBUG [Thread-2055 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:14:26,405 DEBUG [Thread-2055 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:14:26,405 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T20:14:26,411 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=444 (bloomFilter=true), to=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/.tmp/B/ded17ca865fa42bea9593eb489fc8656 2024-11-18T20:14:26,415 INFO [Thread-2059 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2059. 2024-11-18T20:14:26,415 DEBUG [Thread-2059 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-18T20:14:26,415 DEBUG [Thread-2059 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:14:26,415 DEBUG [Thread-2059 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:14:26,415 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T20:14:26,418 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/.tmp/C/0e6cf4459cd641fbbdbbb7047806317b is 50, key is test_row_0/C:col10/1731960864237/Put/seqid=0 2024-11-18T20:14:26,421 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742379_1555 (size=12301) 2024-11-18T20:14:26,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-18T20:14:26,534 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-18T20:14:26,821 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=444 (bloomFilter=true), to=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/.tmp/C/0e6cf4459cd641fbbdbbb7047806317b 2024-11-18T20:14:26,825 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/.tmp/A/4cb386e694d14b3cae02cfc805ffb9f9 as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/A/4cb386e694d14b3cae02cfc805ffb9f9 2024-11-18T20:14:26,828 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/A/4cb386e694d14b3cae02cfc805ffb9f9, entries=150, sequenceid=444, filesize=12.0 K 2024-11-18T20:14:26,829 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/.tmp/B/ded17ca865fa42bea9593eb489fc8656 as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/B/ded17ca865fa42bea9593eb489fc8656 2024-11-18T20:14:26,831 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/B/ded17ca865fa42bea9593eb489fc8656, entries=150, sequenceid=444, filesize=12.0 K 2024-11-18T20:14:26,832 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/.tmp/C/0e6cf4459cd641fbbdbbb7047806317b as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/C/0e6cf4459cd641fbbdbbb7047806317b 2024-11-18T20:14:26,835 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/C/0e6cf4459cd641fbbdbbb7047806317b, entries=150, sequenceid=444, filesize=12.0 K 2024-11-18T20:14:26,836 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=33.54 KB/34350 for 23d90021d1eda01703e7bdf7270f74fc in 1254ms, sequenceid=444, compaction requested=false 2024-11-18T20:14:26,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(2603): Flush status journal for 23d90021d1eda01703e7bdf7270f74fc: 2024-11-18T20:14:26,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc. 2024-11-18T20:14:26,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=140 2024-11-18T20:14:26,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.HMaster(4169): Remote procedure done, pid=140 2024-11-18T20:14:26,838 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=140, resume processing ppid=139 2024-11-18T20:14:26,838 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=140, ppid=139, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.4810 sec 2024-11-18T20:14:26,840 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=139, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=139, table=TestAcidGuarantees in 2.4850 sec 2024-11-18T20:14:28,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-18T20:14:28,513 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-18T20:14:28,513 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-18T20:14:28,514 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 106 2024-11-18T20:14:28,514 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 57 2024-11-18T20:14:28,514 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 29 2024-11-18T20:14:28,514 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 105 2024-11-18T20:14:28,514 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 58 2024-11-18T20:14:28,514 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-18T20:14:28,514 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-18T20:14:28,514 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1701 2024-11-18T20:14:28,514 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 5102 rows 2024-11-18T20:14:28,514 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1698 2024-11-18T20:14:28,514 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 5094 rows 2024-11-18T20:14:28,514 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1685 2024-11-18T20:14:28,514 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 5052 rows 2024-11-18T20:14:28,514 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1706 2024-11-18T20:14:28,514 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 5117 rows 2024-11-18T20:14:28,514 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1700 2024-11-18T20:14:28,514 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 5099 rows 2024-11-18T20:14:28,514 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-18T20:14:28,514 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testScanAtomicity(AcidGuaranteesTestBase.java:112) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-18T20:14:28,514 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:14:28,514 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:14:28,515 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T20:14:28,516 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-18T20:14:28,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] procedure2.ProcedureExecutor(1139): Stored pid=141, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-18T20:14:28,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-18T20:14:28,521 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731960868521"}]},"ts":"1731960868521"} 2024-11-18T20:14:28,524 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-18T20:14:28,524 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-18T20:14:28,525 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=142, ppid=141, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-18T20:14:28,528 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=143, ppid=142, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=23d90021d1eda01703e7bdf7270f74fc, UNASSIGN}] 2024-11-18T20:14:28,529 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=143, ppid=142, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=23d90021d1eda01703e7bdf7270f74fc, UNASSIGN 2024-11-18T20:14:28,530 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=143 updating hbase:meta row=23d90021d1eda01703e7bdf7270f74fc, regionState=CLOSING, regionLocation=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:28,531 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=143, ppid=142, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=23d90021d1eda01703e7bdf7270f74fc, UNASSIGN because future has completed 2024-11-18T20:14:28,531 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-18T20:14:28,531 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=144, ppid=143, state=RUNNABLE, hasLock=false; CloseRegionProcedure 23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223}] 2024-11-18T20:14:28,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-18T20:14:28,684 INFO [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] handler.UnassignRegionHandler(122): Close 23d90021d1eda01703e7bdf7270f74fc 2024-11-18T20:14:28,684 DEBUG [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-18T20:14:28,685 DEBUG [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegion(1722): Closing 23d90021d1eda01703e7bdf7270f74fc, disabling compactions & flushes 2024-11-18T20:14:28,685 INFO [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc. 2024-11-18T20:14:28,685 DEBUG [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc. 2024-11-18T20:14:28,685 DEBUG [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc. after waiting 0 ms 2024-11-18T20:14:28,685 DEBUG [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc. 2024-11-18T20:14:28,685 INFO [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegion(2902): Flushing 23d90021d1eda01703e7bdf7270f74fc 3/3 column families, dataSize=33.54 KB heapSize=88.64 KB 2024-11-18T20:14:28,691 DEBUG [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/.tmp/A/1466bc3050df42acad4026c6870de57b is 50, key is test_row_0/A:col10/1731960866403/Put/seqid=0 2024-11-18T20:14:28,695 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742380_1556 (size=9857) 2024-11-18T20:14:28,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-18T20:14:29,097 INFO [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=455 (bloomFilter=true), to=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/.tmp/A/1466bc3050df42acad4026c6870de57b 2024-11-18T20:14:29,108 DEBUG [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/.tmp/B/ce7391f768e84dfaa9e9113102b18315 is 50, key is test_row_0/B:col10/1731960866403/Put/seqid=0 2024-11-18T20:14:29,112 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742381_1557 (size=9857) 2024-11-18T20:14:29,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-18T20:14:29,513 INFO [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=455 (bloomFilter=true), to=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/.tmp/B/ce7391f768e84dfaa9e9113102b18315 2024-11-18T20:14:29,525 DEBUG [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/.tmp/C/b4a4d849efbb474b9fb592998d7247d6 is 50, key is test_row_0/C:col10/1731960866403/Put/seqid=0 2024-11-18T20:14:29,528 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742382_1558 (size=9857) 2024-11-18T20:14:29,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-18T20:14:29,930 INFO [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=455 (bloomFilter=true), to=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/.tmp/C/b4a4d849efbb474b9fb592998d7247d6 2024-11-18T20:14:29,938 DEBUG [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/.tmp/A/1466bc3050df42acad4026c6870de57b as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/A/1466bc3050df42acad4026c6870de57b 2024-11-18T20:14:29,944 INFO [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/A/1466bc3050df42acad4026c6870de57b, entries=100, sequenceid=455, filesize=9.6 K 2024-11-18T20:14:29,945 DEBUG [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/.tmp/B/ce7391f768e84dfaa9e9113102b18315 as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/B/ce7391f768e84dfaa9e9113102b18315 2024-11-18T20:14:29,950 INFO [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/B/ce7391f768e84dfaa9e9113102b18315, entries=100, sequenceid=455, filesize=9.6 K 2024-11-18T20:14:29,951 DEBUG [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/.tmp/C/b4a4d849efbb474b9fb592998d7247d6 as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/C/b4a4d849efbb474b9fb592998d7247d6 2024-11-18T20:14:29,955 INFO [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/C/b4a4d849efbb474b9fb592998d7247d6, entries=100, sequenceid=455, filesize=9.6 K 2024-11-18T20:14:29,956 INFO [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for 23d90021d1eda01703e7bdf7270f74fc in 1270ms, sequenceid=455, compaction requested=true 2024-11-18T20:14:29,956 DEBUG [StoreCloser-TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/A/6abb4bda81374263ae4e64a9784e7315, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/A/2b481fd570ec4617883560c22a3cd508, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/A/a4912fa3a4e641bba79c237d7b0e5d29, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/A/485401aaa2e542e9867ebcd5cbbc3a65, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/A/ed677c7cbfda479db4cdf6c576e2c22a, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/A/dbbe3593039b40ecb785aabbefcfa7b1, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/A/f37bd9a4a924422fb52549c4e340e073, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/A/0de15de0a61b4d81b7cc2edfcfeb51c0, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/A/f3862c91776c40a48b22b8344ee618e4, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/A/f626567b5b7146e3aca8211a8e562fb7, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/A/aa9fb2625a694b76a14c4815eacac20e, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/A/aa58adf38ed346578f5068ed30c58a60, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/A/c3532bf789364ac99a0e6849f7c6385b, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/A/eca3d1d04c9d47dc81c13429d7a59988, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/A/b21202671a9743f282ebd8a418b05a2f, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/A/b4a146acc8b14703a2150a4f74d4d8b8, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/A/7ca07d43378f4ec2b8259dc478719d45, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/A/4264dbcb302b4f73990ee2afdccfd4ed, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/A/527ecd9bc9544c6283c0b81917c2aee6, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/A/cb61591d6d5543efa1d3f004a444c70f, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/A/314f7a2e4ad344c5816bcb334990585b, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/A/072bcff7b0ea4b9385fec058d55a3d6a, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/A/948983a4aec144a187ef3168e340f4a6, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/A/ea5ba5ce27444144a56e5ed79a073dcf, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/A/c37937f85b9b454b8df620dd1fc2bb4f, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/A/c29b9784d9464777bdb1e2aa57e27495, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/A/ab6ed2a3421a47f2b8bee325c2a5c392, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/A/684367518a0a49e8bca6eb6083817683, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/A/a357e04cd5c94668a56d3e496d92f4aa, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/A/051fb4c1e17b4254a130fc59629ffe48] to archive 2024-11-18T20:14:29,957 DEBUG [StoreCloser-TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-18T20:14:29,959 DEBUG [StoreCloser-TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/A/6abb4bda81374263ae4e64a9784e7315 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/A/6abb4bda81374263ae4e64a9784e7315 2024-11-18T20:14:29,960 DEBUG [StoreCloser-TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/A/2b481fd570ec4617883560c22a3cd508 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/A/2b481fd570ec4617883560c22a3cd508 2024-11-18T20:14:29,961 DEBUG [StoreCloser-TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/A/a4912fa3a4e641bba79c237d7b0e5d29 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/A/a4912fa3a4e641bba79c237d7b0e5d29 2024-11-18T20:14:29,962 DEBUG [StoreCloser-TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/A/485401aaa2e542e9867ebcd5cbbc3a65 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/A/485401aaa2e542e9867ebcd5cbbc3a65 2024-11-18T20:14:29,963 DEBUG [StoreCloser-TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/A/ed677c7cbfda479db4cdf6c576e2c22a to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/A/ed677c7cbfda479db4cdf6c576e2c22a 2024-11-18T20:14:29,965 DEBUG [StoreCloser-TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/A/dbbe3593039b40ecb785aabbefcfa7b1 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/A/dbbe3593039b40ecb785aabbefcfa7b1 2024-11-18T20:14:29,966 DEBUG [StoreCloser-TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/A/f37bd9a4a924422fb52549c4e340e073 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/A/f37bd9a4a924422fb52549c4e340e073 2024-11-18T20:14:29,967 DEBUG [StoreCloser-TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/A/0de15de0a61b4d81b7cc2edfcfeb51c0 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/A/0de15de0a61b4d81b7cc2edfcfeb51c0 2024-11-18T20:14:29,968 DEBUG [StoreCloser-TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/A/f3862c91776c40a48b22b8344ee618e4 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/A/f3862c91776c40a48b22b8344ee618e4 2024-11-18T20:14:29,969 DEBUG [StoreCloser-TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/A/f626567b5b7146e3aca8211a8e562fb7 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/A/f626567b5b7146e3aca8211a8e562fb7 2024-11-18T20:14:29,970 DEBUG [StoreCloser-TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/A/aa9fb2625a694b76a14c4815eacac20e to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/A/aa9fb2625a694b76a14c4815eacac20e 2024-11-18T20:14:29,971 DEBUG [StoreCloser-TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/A/aa58adf38ed346578f5068ed30c58a60 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/A/aa58adf38ed346578f5068ed30c58a60 2024-11-18T20:14:29,973 DEBUG [StoreCloser-TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/A/c3532bf789364ac99a0e6849f7c6385b to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/A/c3532bf789364ac99a0e6849f7c6385b 2024-11-18T20:14:29,974 DEBUG [StoreCloser-TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/A/eca3d1d04c9d47dc81c13429d7a59988 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/A/eca3d1d04c9d47dc81c13429d7a59988 2024-11-18T20:14:29,975 DEBUG [StoreCloser-TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/A/b21202671a9743f282ebd8a418b05a2f to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/A/b21202671a9743f282ebd8a418b05a2f 2024-11-18T20:14:29,976 DEBUG [StoreCloser-TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/A/b4a146acc8b14703a2150a4f74d4d8b8 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/A/b4a146acc8b14703a2150a4f74d4d8b8 2024-11-18T20:14:29,977 DEBUG [StoreCloser-TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/A/7ca07d43378f4ec2b8259dc478719d45 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/A/7ca07d43378f4ec2b8259dc478719d45 2024-11-18T20:14:29,979 DEBUG [StoreCloser-TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/A/4264dbcb302b4f73990ee2afdccfd4ed to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/A/4264dbcb302b4f73990ee2afdccfd4ed 2024-11-18T20:14:29,980 DEBUG [StoreCloser-TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/A/527ecd9bc9544c6283c0b81917c2aee6 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/A/527ecd9bc9544c6283c0b81917c2aee6 2024-11-18T20:14:29,981 DEBUG [StoreCloser-TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/A/cb61591d6d5543efa1d3f004a444c70f to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/A/cb61591d6d5543efa1d3f004a444c70f 2024-11-18T20:14:29,983 DEBUG [StoreCloser-TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/A/314f7a2e4ad344c5816bcb334990585b to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/A/314f7a2e4ad344c5816bcb334990585b 2024-11-18T20:14:29,984 DEBUG [StoreCloser-TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/A/072bcff7b0ea4b9385fec058d55a3d6a to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/A/072bcff7b0ea4b9385fec058d55a3d6a 2024-11-18T20:14:29,985 DEBUG [StoreCloser-TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/A/948983a4aec144a187ef3168e340f4a6 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/A/948983a4aec144a187ef3168e340f4a6 2024-11-18T20:14:29,987 DEBUG [StoreCloser-TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/A/ea5ba5ce27444144a56e5ed79a073dcf to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/A/ea5ba5ce27444144a56e5ed79a073dcf 2024-11-18T20:14:29,988 DEBUG [StoreCloser-TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/A/c37937f85b9b454b8df620dd1fc2bb4f to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/A/c37937f85b9b454b8df620dd1fc2bb4f 2024-11-18T20:14:29,989 DEBUG [StoreCloser-TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/A/c29b9784d9464777bdb1e2aa57e27495 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/A/c29b9784d9464777bdb1e2aa57e27495 2024-11-18T20:14:29,991 DEBUG [StoreCloser-TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/A/ab6ed2a3421a47f2b8bee325c2a5c392 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/A/ab6ed2a3421a47f2b8bee325c2a5c392 2024-11-18T20:14:29,992 DEBUG [StoreCloser-TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/A/684367518a0a49e8bca6eb6083817683 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/A/684367518a0a49e8bca6eb6083817683 2024-11-18T20:14:29,994 DEBUG [StoreCloser-TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/A/a357e04cd5c94668a56d3e496d92f4aa to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/A/a357e04cd5c94668a56d3e496d92f4aa 2024-11-18T20:14:29,995 DEBUG [StoreCloser-TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/A/051fb4c1e17b4254a130fc59629ffe48 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/A/051fb4c1e17b4254a130fc59629ffe48 2024-11-18T20:14:29,997 DEBUG [StoreCloser-TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/B/6197655010b24599a7efc3bc3f3d05d7, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/B/3f2da01bd90a44998ed58b31aea32a0f, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/B/7f701157eb3848bf80bb4c47bb2437bc, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/B/320422ad4af1402c8bd6884965309481, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/B/1be07d98d42a41d68f4ac89fc7a9513a, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/B/5d0d101881774c259e9a9497939c0fdf, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/B/d40c2f043c674d86808c3bbc74972a1e, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/B/99c709c444534070a65020ddb9dcc9ca, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/B/8c1a458ee4d44b2d867154a3a14ea740, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/B/c1a2b0e2648f48858e17aaaef84ce55f, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/B/5588fa102d214bf5bc1720d27c364e70, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/B/1c7bea34baa14059b8bcf482b1f42d3d, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/B/ba25119d8ed745419b3bb21168a659c5, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/B/df3a12917687463cbf9ba225022783bc, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/B/20c2c0e6b3c7463781ad28c4435ce89b, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/B/7fe6ad8f4bea4b379508d438f35ea4fa, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/B/f04cc9ecbf224d8791ec076ae5a8292a, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/B/9e1a71695c5a4db9ba350390b278ffcc, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/B/d3403dff01bf4c90b12e27792941a4f5, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/B/c5d22a92c4094ab2b149173de86be8d3, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/B/11e335fd1bae486aa07c02ad707e8022, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/B/24d2d39e80434108877eecd7d9b554ec, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/B/5734456e867d486697ec69ca3d131044, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/B/167b31563e074e498fbd5e79b99f6954, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/B/3a9791a87f974a959254d83efffbba24, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/B/d798bf9a6f2148bc9e2e4b29f4e841fe, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/B/9f5e9e1fdbbd4af2b6530f36483166c3, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/B/8922649520834958b53a7bbbcd7fb257, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/B/5de4f933d11049d788fea5a2edeb5285, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/B/d7115417c82e4375924463412f23eb99] to archive 2024-11-18T20:14:29,998 DEBUG [StoreCloser-TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-18T20:14:30,000 DEBUG [StoreCloser-TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/B/6197655010b24599a7efc3bc3f3d05d7 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/B/6197655010b24599a7efc3bc3f3d05d7 2024-11-18T20:14:30,002 DEBUG [StoreCloser-TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/B/3f2da01bd90a44998ed58b31aea32a0f to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/B/3f2da01bd90a44998ed58b31aea32a0f 2024-11-18T20:14:30,003 DEBUG [StoreCloser-TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/B/7f701157eb3848bf80bb4c47bb2437bc to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/B/7f701157eb3848bf80bb4c47bb2437bc 2024-11-18T20:14:30,005 DEBUG [StoreCloser-TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/B/320422ad4af1402c8bd6884965309481 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/B/320422ad4af1402c8bd6884965309481 2024-11-18T20:14:30,006 DEBUG [StoreCloser-TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/B/1be07d98d42a41d68f4ac89fc7a9513a to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/B/1be07d98d42a41d68f4ac89fc7a9513a 2024-11-18T20:14:30,007 DEBUG [StoreCloser-TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/B/5d0d101881774c259e9a9497939c0fdf to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/B/5d0d101881774c259e9a9497939c0fdf 2024-11-18T20:14:30,009 DEBUG [StoreCloser-TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/B/d40c2f043c674d86808c3bbc74972a1e to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/B/d40c2f043c674d86808c3bbc74972a1e 2024-11-18T20:14:30,010 DEBUG [StoreCloser-TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/B/99c709c444534070a65020ddb9dcc9ca to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/B/99c709c444534070a65020ddb9dcc9ca 2024-11-18T20:14:30,011 DEBUG [StoreCloser-TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/B/8c1a458ee4d44b2d867154a3a14ea740 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/B/8c1a458ee4d44b2d867154a3a14ea740 2024-11-18T20:14:30,013 DEBUG [StoreCloser-TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/B/c1a2b0e2648f48858e17aaaef84ce55f to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/B/c1a2b0e2648f48858e17aaaef84ce55f 2024-11-18T20:14:30,014 DEBUG [StoreCloser-TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/B/5588fa102d214bf5bc1720d27c364e70 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/B/5588fa102d214bf5bc1720d27c364e70 2024-11-18T20:14:30,014 DEBUG [StoreCloser-TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/B/1c7bea34baa14059b8bcf482b1f42d3d to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/B/1c7bea34baa14059b8bcf482b1f42d3d 2024-11-18T20:14:30,015 DEBUG [StoreCloser-TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/B/ba25119d8ed745419b3bb21168a659c5 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/B/ba25119d8ed745419b3bb21168a659c5 2024-11-18T20:14:30,016 DEBUG [StoreCloser-TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/B/df3a12917687463cbf9ba225022783bc to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/B/df3a12917687463cbf9ba225022783bc 2024-11-18T20:14:30,016 DEBUG [StoreCloser-TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/B/20c2c0e6b3c7463781ad28c4435ce89b to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/B/20c2c0e6b3c7463781ad28c4435ce89b 2024-11-18T20:14:30,017 DEBUG [StoreCloser-TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/B/7fe6ad8f4bea4b379508d438f35ea4fa to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/B/7fe6ad8f4bea4b379508d438f35ea4fa 2024-11-18T20:14:30,018 DEBUG [StoreCloser-TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/B/f04cc9ecbf224d8791ec076ae5a8292a to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/B/f04cc9ecbf224d8791ec076ae5a8292a 2024-11-18T20:14:30,018 DEBUG [StoreCloser-TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/B/9e1a71695c5a4db9ba350390b278ffcc to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/B/9e1a71695c5a4db9ba350390b278ffcc 2024-11-18T20:14:30,019 DEBUG [StoreCloser-TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/B/d3403dff01bf4c90b12e27792941a4f5 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/B/d3403dff01bf4c90b12e27792941a4f5 2024-11-18T20:14:30,020 DEBUG [StoreCloser-TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/B/c5d22a92c4094ab2b149173de86be8d3 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/B/c5d22a92c4094ab2b149173de86be8d3 2024-11-18T20:14:30,021 DEBUG [StoreCloser-TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/B/11e335fd1bae486aa07c02ad707e8022 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/B/11e335fd1bae486aa07c02ad707e8022 2024-11-18T20:14:30,021 DEBUG [StoreCloser-TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/B/24d2d39e80434108877eecd7d9b554ec to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/B/24d2d39e80434108877eecd7d9b554ec 2024-11-18T20:14:30,022 DEBUG [StoreCloser-TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/B/5734456e867d486697ec69ca3d131044 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/B/5734456e867d486697ec69ca3d131044 2024-11-18T20:14:30,023 DEBUG [StoreCloser-TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/B/167b31563e074e498fbd5e79b99f6954 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/B/167b31563e074e498fbd5e79b99f6954 2024-11-18T20:14:30,023 DEBUG [StoreCloser-TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/B/3a9791a87f974a959254d83efffbba24 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/B/3a9791a87f974a959254d83efffbba24 2024-11-18T20:14:30,024 DEBUG [StoreCloser-TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/B/d798bf9a6f2148bc9e2e4b29f4e841fe to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/B/d798bf9a6f2148bc9e2e4b29f4e841fe 2024-11-18T20:14:30,025 DEBUG [StoreCloser-TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/B/9f5e9e1fdbbd4af2b6530f36483166c3 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/B/9f5e9e1fdbbd4af2b6530f36483166c3 2024-11-18T20:14:30,026 DEBUG [StoreCloser-TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/B/8922649520834958b53a7bbbcd7fb257 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/B/8922649520834958b53a7bbbcd7fb257 2024-11-18T20:14:30,026 DEBUG [StoreCloser-TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/B/5de4f933d11049d788fea5a2edeb5285 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/B/5de4f933d11049d788fea5a2edeb5285 2024-11-18T20:14:30,027 DEBUG [StoreCloser-TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/B/d7115417c82e4375924463412f23eb99 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/B/d7115417c82e4375924463412f23eb99 2024-11-18T20:14:30,028 DEBUG [StoreCloser-TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/C/658015f09d4547fe95f8b60bdb49d975, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/C/dee21cd8e9ca4b3e9b48b4a500324af2, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/C/a67f78075b544420a031fe3472f73b5f, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/C/73dd2270193d4a0eb5977663ad0c9c57, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/C/d8ae2617f2e24a0c9f7853d4f71f70a3, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/C/58f7a3be119d4201af7f9b9c8c0fe99d, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/C/ded89bf64a10475ca371570944ef60a9, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/C/871bd0b0d0cd4d2ea67922e7a41fa62b, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/C/dd74de3ef0734a72a3e6043697fb745e, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/C/2ff61129268341d3a3b634a2a6eb686b, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/C/f79bc7fc9d474d2d85d75f490c15b551, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/C/90eb122cce754bfab78929f91785afeb, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/C/b1e55172024f4005af135a2cd54f72d8, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/C/9104fa5f16e54055935b21e2eeaffa45, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/C/c833eb9b468546dbb7f321cdf01d8767, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/C/f07ed15ec0d54cf2b03f54f59b562fc4, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/C/2df50040410a49a0ba45e6139fdd1a88, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/C/c6c190361b4a40908808f7757fbf4d7d, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/C/4273a1dead234073a9f9a921b2e591cb, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/C/2be8cc2e5cf24b09a8e5455f9021590c, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/C/4e4de7e5bd99472e9841c2bdb3759f9d, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/C/9df14f10b8134882b9ed3fa9f1976ca8, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/C/4fb476bab21e4462a958d9bd9b563251, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/C/79279deee29e45c381a0a64b8afa5ec0, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/C/cd713709f7b348b78cf489b5313a50c3, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/C/49db5d906b88433ba35319adc4b7bcb5, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/C/0eb3fc69063f4efb8562407794df502a, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/C/8008d6970185499784364a1ae6ce1e69, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/C/5ed42a638aa1426a973974e899dc69b0, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/C/9f18f3f38b304a058486899db8dbed2a] to archive 2024-11-18T20:14:30,029 DEBUG [StoreCloser-TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-18T20:14:30,030 DEBUG [StoreCloser-TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/C/658015f09d4547fe95f8b60bdb49d975 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/C/658015f09d4547fe95f8b60bdb49d975 2024-11-18T20:14:30,031 DEBUG [StoreCloser-TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/C/dee21cd8e9ca4b3e9b48b4a500324af2 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/C/dee21cd8e9ca4b3e9b48b4a500324af2 2024-11-18T20:14:30,032 DEBUG [StoreCloser-TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/C/a67f78075b544420a031fe3472f73b5f to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/C/a67f78075b544420a031fe3472f73b5f 2024-11-18T20:14:30,033 DEBUG [StoreCloser-TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/C/73dd2270193d4a0eb5977663ad0c9c57 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/C/73dd2270193d4a0eb5977663ad0c9c57 2024-11-18T20:14:30,033 DEBUG [StoreCloser-TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/C/d8ae2617f2e24a0c9f7853d4f71f70a3 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/C/d8ae2617f2e24a0c9f7853d4f71f70a3 2024-11-18T20:14:30,034 DEBUG [StoreCloser-TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/C/58f7a3be119d4201af7f9b9c8c0fe99d to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/C/58f7a3be119d4201af7f9b9c8c0fe99d 2024-11-18T20:14:30,035 DEBUG [StoreCloser-TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/C/ded89bf64a10475ca371570944ef60a9 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/C/ded89bf64a10475ca371570944ef60a9 2024-11-18T20:14:30,035 DEBUG [StoreCloser-TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/C/871bd0b0d0cd4d2ea67922e7a41fa62b to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/C/871bd0b0d0cd4d2ea67922e7a41fa62b 2024-11-18T20:14:30,036 DEBUG [StoreCloser-TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/C/dd74de3ef0734a72a3e6043697fb745e to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/C/dd74de3ef0734a72a3e6043697fb745e 2024-11-18T20:14:30,037 DEBUG [StoreCloser-TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/C/2ff61129268341d3a3b634a2a6eb686b to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/C/2ff61129268341d3a3b634a2a6eb686b 2024-11-18T20:14:30,037 DEBUG [StoreCloser-TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/C/f79bc7fc9d474d2d85d75f490c15b551 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/C/f79bc7fc9d474d2d85d75f490c15b551 2024-11-18T20:14:30,038 DEBUG [StoreCloser-TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/C/90eb122cce754bfab78929f91785afeb to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/C/90eb122cce754bfab78929f91785afeb 2024-11-18T20:14:30,039 DEBUG [StoreCloser-TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/C/b1e55172024f4005af135a2cd54f72d8 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/C/b1e55172024f4005af135a2cd54f72d8 2024-11-18T20:14:30,040 DEBUG [StoreCloser-TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/C/9104fa5f16e54055935b21e2eeaffa45 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/C/9104fa5f16e54055935b21e2eeaffa45 2024-11-18T20:14:30,040 DEBUG [StoreCloser-TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/C/c833eb9b468546dbb7f321cdf01d8767 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/C/c833eb9b468546dbb7f321cdf01d8767 2024-11-18T20:14:30,041 DEBUG [StoreCloser-TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/C/f07ed15ec0d54cf2b03f54f59b562fc4 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/C/f07ed15ec0d54cf2b03f54f59b562fc4 2024-11-18T20:14:30,042 DEBUG [StoreCloser-TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/C/2df50040410a49a0ba45e6139fdd1a88 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/C/2df50040410a49a0ba45e6139fdd1a88 2024-11-18T20:14:30,043 DEBUG [StoreCloser-TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/C/c6c190361b4a40908808f7757fbf4d7d to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/C/c6c190361b4a40908808f7757fbf4d7d 2024-11-18T20:14:30,044 DEBUG [StoreCloser-TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/C/4273a1dead234073a9f9a921b2e591cb to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/C/4273a1dead234073a9f9a921b2e591cb 2024-11-18T20:14:30,045 DEBUG [StoreCloser-TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/C/2be8cc2e5cf24b09a8e5455f9021590c to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/C/2be8cc2e5cf24b09a8e5455f9021590c 2024-11-18T20:14:30,045 DEBUG [StoreCloser-TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/C/4e4de7e5bd99472e9841c2bdb3759f9d to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/C/4e4de7e5bd99472e9841c2bdb3759f9d 2024-11-18T20:14:30,046 DEBUG [StoreCloser-TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/C/9df14f10b8134882b9ed3fa9f1976ca8 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/C/9df14f10b8134882b9ed3fa9f1976ca8 2024-11-18T20:14:30,047 DEBUG [StoreCloser-TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/C/4fb476bab21e4462a958d9bd9b563251 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/C/4fb476bab21e4462a958d9bd9b563251 2024-11-18T20:14:30,047 DEBUG [StoreCloser-TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/C/79279deee29e45c381a0a64b8afa5ec0 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/C/79279deee29e45c381a0a64b8afa5ec0 2024-11-18T20:14:30,048 DEBUG [StoreCloser-TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/C/cd713709f7b348b78cf489b5313a50c3 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/C/cd713709f7b348b78cf489b5313a50c3 2024-11-18T20:14:30,049 DEBUG [StoreCloser-TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/C/49db5d906b88433ba35319adc4b7bcb5 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/C/49db5d906b88433ba35319adc4b7bcb5 2024-11-18T20:14:30,050 DEBUG [StoreCloser-TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/C/0eb3fc69063f4efb8562407794df502a to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/C/0eb3fc69063f4efb8562407794df502a 2024-11-18T20:14:30,051 DEBUG [StoreCloser-TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/C/8008d6970185499784364a1ae6ce1e69 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/C/8008d6970185499784364a1ae6ce1e69 2024-11-18T20:14:30,052 DEBUG [StoreCloser-TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/C/5ed42a638aa1426a973974e899dc69b0 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/C/5ed42a638aa1426a973974e899dc69b0 2024-11-18T20:14:30,052 DEBUG [StoreCloser-TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/C/9f18f3f38b304a058486899db8dbed2a to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/C/9f18f3f38b304a058486899db8dbed2a 2024-11-18T20:14:30,056 DEBUG [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/recovered.edits/458.seqid, newMaxSeqId=458, maxSeqId=1 2024-11-18T20:14:30,056 INFO [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc. 2024-11-18T20:14:30,056 DEBUG [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegion(1676): Region close journal for 23d90021d1eda01703e7bdf7270f74fc: Waiting for close lock at 1731960868684Running coprocessor pre-close hooks at 1731960868684Disabling compacts and flushes for region at 1731960868684Disabling writes for close at 1731960868685 (+1 ms)Obtaining lock to block concurrent updates at 1731960868685Preparing flush snapshotting stores in 23d90021d1eda01703e7bdf7270f74fc at 1731960868685Finished memstore snapshotting TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=90720, getOffHeapSize=0, getCellsCount=750 at 1731960868686 (+1 ms)Flushing stores of TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc. at 1731960868687 (+1 ms)Flushing 23d90021d1eda01703e7bdf7270f74fc/A: creating writer at 1731960868687Flushing 23d90021d1eda01703e7bdf7270f74fc/A: appending metadata at 1731960868691 (+4 ms)Flushing 23d90021d1eda01703e7bdf7270f74fc/A: closing flushed file at 1731960868691Flushing 23d90021d1eda01703e7bdf7270f74fc/B: creating writer at 1731960869103 (+412 ms)Flushing 23d90021d1eda01703e7bdf7270f74fc/B: appending metadata at 1731960869107 (+4 ms)Flushing 23d90021d1eda01703e7bdf7270f74fc/B: closing flushed file at 1731960869107Flushing 23d90021d1eda01703e7bdf7270f74fc/C: creating writer at 1731960869521 (+414 ms)Flushing 23d90021d1eda01703e7bdf7270f74fc/C: appending metadata at 1731960869525 (+4 ms)Flushing 23d90021d1eda01703e7bdf7270f74fc/C: closing flushed file at 1731960869525Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@79e925b4: reopening flushed file at 1731960869936 (+411 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@52f651ba: reopening flushed file at 1731960869944 (+8 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1004faf5: reopening flushed file at 1731960869950 (+6 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for 23d90021d1eda01703e7bdf7270f74fc in 1270ms, sequenceid=455, compaction requested=true at 1731960869956 (+6 ms)Writing region close event to WAL at 1731960870054 (+98 ms)Running coprocessor post-close hooks at 1731960870056 (+2 ms)Closed at 1731960870056 2024-11-18T20:14:30,058 INFO [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] handler.UnassignRegionHandler(157): Closed 23d90021d1eda01703e7bdf7270f74fc 2024-11-18T20:14:30,059 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=143 updating hbase:meta row=23d90021d1eda01703e7bdf7270f74fc, regionState=CLOSED 2024-11-18T20:14:30,059 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=144, ppid=143, state=RUNNABLE, hasLock=false; CloseRegionProcedure 23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 because future has completed 2024-11-18T20:14:30,061 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=144, resume processing ppid=143 2024-11-18T20:14:30,061 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=144, ppid=143, state=SUCCESS, hasLock=false; CloseRegionProcedure 23d90021d1eda01703e7bdf7270f74fc, server=c0a89b2656d4,35963,1731960718223 in 1.5290 sec 2024-11-18T20:14:30,063 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=143, resume processing ppid=142 2024-11-18T20:14:30,063 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=143, ppid=142, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=23d90021d1eda01703e7bdf7270f74fc, UNASSIGN in 1.5330 sec 2024-11-18T20:14:30,064 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=142, resume processing ppid=141 2024-11-18T20:14:30,064 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=142, ppid=141, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.5380 sec 2024-11-18T20:14:30,065 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731960870065"}]},"ts":"1731960870065"} 2024-11-18T20:14:30,066 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-18T20:14:30,066 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-18T20:14:30,068 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=141, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.5510 sec 2024-11-18T20:14:30,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-18T20:14:30,663 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-18T20:14:30,664 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-18T20:14:30,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] procedure2.ProcedureExecutor(1139): Stored pid=145, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-18T20:14:30,667 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=145, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-18T20:14:30,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=145 2024-11-18T20:14:30,668 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=145, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-18T20:14:30,671 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc 2024-11-18T20:14:30,674 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/A, FileablePath, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/B, FileablePath, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/C, FileablePath, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/recovered.edits] 2024-11-18T20:14:30,678 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/A/1466bc3050df42acad4026c6870de57b to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/A/1466bc3050df42acad4026c6870de57b 2024-11-18T20:14:30,680 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/A/4cb386e694d14b3cae02cfc805ffb9f9 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/A/4cb386e694d14b3cae02cfc805ffb9f9 2024-11-18T20:14:30,681 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/A/5d9ed599fb2c4d30a726d81af79c3c20 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/A/5d9ed599fb2c4d30a726d81af79c3c20 2024-11-18T20:14:30,684 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/B/a366f8eda60840f2873deb2a8f979083 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/B/a366f8eda60840f2873deb2a8f979083 2024-11-18T20:14:30,686 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/B/ce7391f768e84dfaa9e9113102b18315 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/B/ce7391f768e84dfaa9e9113102b18315 2024-11-18T20:14:30,687 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/B/ded17ca865fa42bea9593eb489fc8656 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/B/ded17ca865fa42bea9593eb489fc8656 2024-11-18T20:14:30,688 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/C/0e6cf4459cd641fbbdbbb7047806317b to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/C/0e6cf4459cd641fbbdbbb7047806317b 2024-11-18T20:14:30,689 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/C/24ee05f43a0544d5925ec842af646cd2 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/C/24ee05f43a0544d5925ec842af646cd2 2024-11-18T20:14:30,689 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/C/b4a4d849efbb474b9fb592998d7247d6 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/C/b4a4d849efbb474b9fb592998d7247d6 2024-11-18T20:14:30,691 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/recovered.edits/458.seqid to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc/recovered.edits/458.seqid 2024-11-18T20:14:30,691 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/23d90021d1eda01703e7bdf7270f74fc 2024-11-18T20:14:30,691 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-18T20:14:30,693 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=145, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-18T20:14:30,694 WARN [PEWorker-3 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-18T20:14:30,696 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-18T20:14:30,697 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=145, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-18T20:14:30,697 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-18T20:14:30,697 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731960870697"}]},"ts":"9223372036854775807"} 2024-11-18T20:14:30,699 INFO [PEWorker-3 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-18T20:14:30,699 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 23d90021d1eda01703e7bdf7270f74fc, NAME => 'TestAcidGuarantees,,1731960842808.23d90021d1eda01703e7bdf7270f74fc.', STARTKEY => '', ENDKEY => ''}] 2024-11-18T20:14:30,699 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-18T20:14:30,699 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731960870699"}]},"ts":"9223372036854775807"} 2024-11-18T20:14:30,701 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-18T20:14:30,702 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(133): Finished pid=145, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-18T20:14:30,704 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=145, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 38 msec 2024-11-18T20:14:30,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=145 2024-11-18T20:14:30,773 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-18T20:14:30,776 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-18T20:14:30,782 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testScanAtomicity Thread=237 (was 237), OpenFileDescriptor=449 (was 445) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=443 (was 454), ProcessCount=11 (was 11), AvailableMemoryMB=2446 (was 2493) 2024-11-18T20:14:30,790 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testMobGetAtomicity Thread=237, OpenFileDescriptor=449, MaxFileDescriptor=1048576, SystemLoadAverage=443, ProcessCount=11, AvailableMemoryMB=2446 2024-11-18T20:14:30,791 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-18T20:14:30,791 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-18T20:14:30,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] procedure2.ProcedureExecutor(1139): Stored pid=146, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-18T20:14:30,793 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=146, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-18T20:14:30,793 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:30,793 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 146 2024-11-18T20:14:30,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-18T20:14:30,794 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=146, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-18T20:14:30,798 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742383_1559 (size=959) 2024-11-18T20:14:30,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-18T20:14:31,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-18T20:14:31,205 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 3c05f66f0b4334a1f317161bff1a01e6, NAME => 'TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123 2024-11-18T20:14:31,210 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742384_1560 (size=53) 2024-11-18T20:14:31,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-18T20:14:31,613 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T20:14:31,613 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 3c05f66f0b4334a1f317161bff1a01e6, disabling compactions & flushes 2024-11-18T20:14:31,613 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6. 2024-11-18T20:14:31,613 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6. 2024-11-18T20:14:31,613 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6. after waiting 0 ms 2024-11-18T20:14:31,613 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6. 2024-11-18T20:14:31,613 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6. 2024-11-18T20:14:31,613 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 3c05f66f0b4334a1f317161bff1a01e6: Waiting for close lock at 1731960871613Disabling compacts and flushes for region at 1731960871613Disabling writes for close at 1731960871613Writing region close event to WAL at 1731960871613Closed at 1731960871613 2024-11-18T20:14:31,616 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=146, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-18T20:14:31,617 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731960871616"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731960871616"}]},"ts":"1731960871616"} 2024-11-18T20:14:31,619 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-18T20:14:31,621 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=146, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-18T20:14:31,621 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731960871621"}]},"ts":"1731960871621"} 2024-11-18T20:14:31,623 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-18T20:14:31,624 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=147, ppid=146, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=3c05f66f0b4334a1f317161bff1a01e6, ASSIGN}] 2024-11-18T20:14:31,625 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=147, ppid=146, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=3c05f66f0b4334a1f317161bff1a01e6, ASSIGN 2024-11-18T20:14:31,627 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=147, ppid=146, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=3c05f66f0b4334a1f317161bff1a01e6, ASSIGN; state=OFFLINE, location=c0a89b2656d4,35963,1731960718223; forceNewPlan=false, retain=false 2024-11-18T20:14:31,777 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=147 updating hbase:meta row=3c05f66f0b4334a1f317161bff1a01e6, regionState=OPENING, regionLocation=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:31,779 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=147, ppid=146, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=3c05f66f0b4334a1f317161bff1a01e6, ASSIGN because future has completed 2024-11-18T20:14:31,779 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=148, ppid=147, state=RUNNABLE, hasLock=false; OpenRegionProcedure 3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223}] 2024-11-18T20:14:31,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-18T20:14:31,937 INFO [RS_OPEN_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6. 2024-11-18T20:14:31,937 DEBUG [RS_OPEN_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] regionserver.HRegion(7752): Opening region: {ENCODED => 3c05f66f0b4334a1f317161bff1a01e6, NAME => 'TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6.', STARTKEY => '', ENDKEY => ''} 2024-11-18T20:14:31,938 DEBUG [RS_OPEN_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 3c05f66f0b4334a1f317161bff1a01e6 2024-11-18T20:14:31,938 DEBUG [RS_OPEN_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T20:14:31,938 DEBUG [RS_OPEN_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] regionserver.HRegion(7794): checking encryption for 3c05f66f0b4334a1f317161bff1a01e6 2024-11-18T20:14:31,938 DEBUG [RS_OPEN_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] regionserver.HRegion(7797): checking classloading for 3c05f66f0b4334a1f317161bff1a01e6 2024-11-18T20:14:31,940 INFO [StoreOpener-3c05f66f0b4334a1f317161bff1a01e6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 3c05f66f0b4334a1f317161bff1a01e6 2024-11-18T20:14:31,942 INFO [StoreOpener-3c05f66f0b4334a1f317161bff1a01e6-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3c05f66f0b4334a1f317161bff1a01e6 columnFamilyName A 2024-11-18T20:14:31,942 DEBUG [StoreOpener-3c05f66f0b4334a1f317161bff1a01e6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:31,943 INFO [StoreOpener-3c05f66f0b4334a1f317161bff1a01e6-1 {}] regionserver.HStore(327): Store=3c05f66f0b4334a1f317161bff1a01e6/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T20:14:31,943 INFO [StoreOpener-3c05f66f0b4334a1f317161bff1a01e6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 3c05f66f0b4334a1f317161bff1a01e6 2024-11-18T20:14:31,944 INFO [StoreOpener-3c05f66f0b4334a1f317161bff1a01e6-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3c05f66f0b4334a1f317161bff1a01e6 columnFamilyName B 2024-11-18T20:14:31,944 DEBUG [StoreOpener-3c05f66f0b4334a1f317161bff1a01e6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:31,945 INFO [StoreOpener-3c05f66f0b4334a1f317161bff1a01e6-1 {}] regionserver.HStore(327): Store=3c05f66f0b4334a1f317161bff1a01e6/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T20:14:31,945 INFO [StoreOpener-3c05f66f0b4334a1f317161bff1a01e6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 3c05f66f0b4334a1f317161bff1a01e6 2024-11-18T20:14:31,946 INFO [StoreOpener-3c05f66f0b4334a1f317161bff1a01e6-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3c05f66f0b4334a1f317161bff1a01e6 columnFamilyName C 2024-11-18T20:14:31,946 DEBUG [StoreOpener-3c05f66f0b4334a1f317161bff1a01e6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:31,946 INFO [StoreOpener-3c05f66f0b4334a1f317161bff1a01e6-1 {}] regionserver.HStore(327): Store=3c05f66f0b4334a1f317161bff1a01e6/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T20:14:31,946 DEBUG [RS_OPEN_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] regionserver.HRegion(1038): replaying wal for 3c05f66f0b4334a1f317161bff1a01e6 2024-11-18T20:14:31,947 DEBUG [RS_OPEN_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6 2024-11-18T20:14:31,947 DEBUG [RS_OPEN_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6 2024-11-18T20:14:31,948 DEBUG [RS_OPEN_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] regionserver.HRegion(1048): stopping wal replay for 3c05f66f0b4334a1f317161bff1a01e6 2024-11-18T20:14:31,948 DEBUG [RS_OPEN_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] regionserver.HRegion(1060): Cleaning up temporary data for 3c05f66f0b4334a1f317161bff1a01e6 2024-11-18T20:14:31,949 DEBUG [RS_OPEN_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-18T20:14:31,949 DEBUG [RS_OPEN_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] regionserver.HRegion(1093): writing seq id for 3c05f66f0b4334a1f317161bff1a01e6 2024-11-18T20:14:31,951 DEBUG [RS_OPEN_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-18T20:14:31,951 INFO [RS_OPEN_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] regionserver.HRegion(1114): Opened 3c05f66f0b4334a1f317161bff1a01e6; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68877369, jitterRate=0.02635277807712555}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-18T20:14:31,951 DEBUG [RS_OPEN_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 3c05f66f0b4334a1f317161bff1a01e6 2024-11-18T20:14:31,951 DEBUG [RS_OPEN_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] regionserver.HRegion(1006): Region open journal for 3c05f66f0b4334a1f317161bff1a01e6: Running coprocessor pre-open hook at 1731960871939Writing region info on filesystem at 1731960871939Initializing all the Stores at 1731960871940 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731960871940Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731960871940Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731960871940Cleaning up temporary data from old regions at 1731960871948 (+8 ms)Running coprocessor post-open hooks at 1731960871951 (+3 ms)Region opened successfully at 1731960871951 2024-11-18T20:14:31,952 INFO [RS_OPEN_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., pid=148, masterSystemTime=1731960871931 2024-11-18T20:14:31,953 DEBUG [RS_OPEN_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6. 2024-11-18T20:14:31,953 INFO [RS_OPEN_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6. 2024-11-18T20:14:31,954 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=147 updating hbase:meta row=3c05f66f0b4334a1f317161bff1a01e6, regionState=OPEN, openSeqNum=2, regionLocation=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:31,955 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=148, ppid=147, state=RUNNABLE, hasLock=false; OpenRegionProcedure 3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 because future has completed 2024-11-18T20:14:31,957 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=148, resume processing ppid=147 2024-11-18T20:14:31,957 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=148, ppid=147, state=SUCCESS, hasLock=false; OpenRegionProcedure 3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 in 177 msec 2024-11-18T20:14:31,959 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=147, resume processing ppid=146 2024-11-18T20:14:31,959 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=147, ppid=146, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=3c05f66f0b4334a1f317161bff1a01e6, ASSIGN in 333 msec 2024-11-18T20:14:31,959 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=146, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-18T20:14:31,960 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731960871959"}]},"ts":"1731960871959"} 2024-11-18T20:14:31,961 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-18T20:14:31,961 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=146, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-18T20:14:31,962 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=146, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1700 sec 2024-11-18T20:14:32,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-18T20:14:32,944 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-18T20:14:32,946 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@109f3221, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:14:32,946 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c0a89b2656d4,41277,-1 for getting cluster id 2024-11-18T20:14:32,946 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T20:14:32,948 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ac99df6b-7791-4693-a81a-6a5a39504490' 2024-11-18T20:14:32,948 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T20:14:32,949 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ac99df6b-7791-4693-a81a-6a5a39504490" 2024-11-18T20:14:32,949 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@20185a67, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:14:32,949 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c0a89b2656d4,41277,-1] 2024-11-18T20:14:32,949 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T20:14:32,950 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:14:32,951 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54450, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T20:14:32,952 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@622625f8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:14:32,952 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T20:14:32,953 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c0a89b2656d4,35963,1731960718223, seqNum=-1] 2024-11-18T20:14:32,954 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T20:14:32,955 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51780, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T20:14:32,955 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-18T20:14:32,956 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncConnectionImpl(321): The fetched master address is c0a89b2656d4,41277,1731960717534 2024-11-18T20:14:32,956 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@3ace0057 2024-11-18T20:14:32,956 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-18T20:14:32,957 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54454, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-18T20:14:32,959 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-18T20:14:32,959 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-18T20:14:32,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] procedure2.ProcedureExecutor(1139): Stored pid=149, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-18T20:14:32,973 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742385_1561 (size=995) 2024-11-18T20:14:33,375 DEBUG [PEWorker-2 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.959 2024-11-18T20:14:33,376 INFO [PEWorker-2 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.995 2024-11-18T20:14:33,379 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=150, ppid=149, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-18T20:14:33,383 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=151, ppid=150, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=3c05f66f0b4334a1f317161bff1a01e6, REOPEN/MOVE}] 2024-11-18T20:14:33,384 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=151, ppid=150, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=3c05f66f0b4334a1f317161bff1a01e6, REOPEN/MOVE 2024-11-18T20:14:33,385 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=151 updating hbase:meta row=3c05f66f0b4334a1f317161bff1a01e6, regionState=CLOSING, regionLocation=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:33,387 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=151, ppid=150, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=3c05f66f0b4334a1f317161bff1a01e6, REOPEN/MOVE because future has completed 2024-11-18T20:14:33,387 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-18T20:14:33,387 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=152, ppid=151, state=RUNNABLE, hasLock=false; CloseRegionProcedure 3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223}] 2024-11-18T20:14:33,540 INFO [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=152}] handler.UnassignRegionHandler(122): Close 3c05f66f0b4334a1f317161bff1a01e6 2024-11-18T20:14:33,540 DEBUG [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=152}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-18T20:14:33,540 DEBUG [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=152}] regionserver.HRegion(1722): Closing 3c05f66f0b4334a1f317161bff1a01e6, disabling compactions & flushes 2024-11-18T20:14:33,540 INFO [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=152}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6. 2024-11-18T20:14:33,540 DEBUG [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=152}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6. 2024-11-18T20:14:33,540 DEBUG [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=152}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6. after waiting 0 ms 2024-11-18T20:14:33,541 DEBUG [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=152}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6. 2024-11-18T20:14:33,547 DEBUG [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=152}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-18T20:14:33,548 INFO [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=152}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6. 2024-11-18T20:14:33,549 DEBUG [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=152}] regionserver.HRegion(1676): Region close journal for 3c05f66f0b4334a1f317161bff1a01e6: Waiting for close lock at 1731960873540Running coprocessor pre-close hooks at 1731960873540Disabling compacts and flushes for region at 1731960873540Disabling writes for close at 1731960873541 (+1 ms)Writing region close event to WAL at 1731960873541Running coprocessor post-close hooks at 1731960873548 (+7 ms)Closed at 1731960873548 2024-11-18T20:14:33,549 WARN [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=152}] regionserver.HRegionServer(3299): Not adding moved region record: 3c05f66f0b4334a1f317161bff1a01e6 to self. 2024-11-18T20:14:33,552 INFO [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=152}] handler.UnassignRegionHandler(157): Closed 3c05f66f0b4334a1f317161bff1a01e6 2024-11-18T20:14:33,553 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=151 updating hbase:meta row=3c05f66f0b4334a1f317161bff1a01e6, regionState=CLOSED 2024-11-18T20:14:33,555 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=152, ppid=151, state=RUNNABLE, hasLock=false; CloseRegionProcedure 3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 because future has completed 2024-11-18T20:14:33,557 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=152, resume processing ppid=151 2024-11-18T20:14:33,557 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=152, ppid=151, state=SUCCESS, hasLock=false; CloseRegionProcedure 3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 in 168 msec 2024-11-18T20:14:33,558 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=151, ppid=150, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=3c05f66f0b4334a1f317161bff1a01e6, REOPEN/MOVE; state=CLOSED, location=c0a89b2656d4,35963,1731960718223; forceNewPlan=false, retain=true 2024-11-18T20:14:33,709 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=151 updating hbase:meta row=3c05f66f0b4334a1f317161bff1a01e6, regionState=OPENING, regionLocation=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:33,712 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=151, ppid=150, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=3c05f66f0b4334a1f317161bff1a01e6, REOPEN/MOVE because future has completed 2024-11-18T20:14:33,712 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=153, ppid=151, state=RUNNABLE, hasLock=false; OpenRegionProcedure 3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223}] 2024-11-18T20:14:33,871 INFO [RS_OPEN_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6. 2024-11-18T20:14:33,871 DEBUG [RS_OPEN_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] regionserver.HRegion(7752): Opening region: {ENCODED => 3c05f66f0b4334a1f317161bff1a01e6, NAME => 'TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6.', STARTKEY => '', ENDKEY => ''} 2024-11-18T20:14:33,872 DEBUG [RS_OPEN_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 3c05f66f0b4334a1f317161bff1a01e6 2024-11-18T20:14:33,872 DEBUG [RS_OPEN_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T20:14:33,872 DEBUG [RS_OPEN_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] regionserver.HRegion(7794): checking encryption for 3c05f66f0b4334a1f317161bff1a01e6 2024-11-18T20:14:33,872 DEBUG [RS_OPEN_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] regionserver.HRegion(7797): checking classloading for 3c05f66f0b4334a1f317161bff1a01e6 2024-11-18T20:14:33,874 INFO [StoreOpener-3c05f66f0b4334a1f317161bff1a01e6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 3c05f66f0b4334a1f317161bff1a01e6 2024-11-18T20:14:33,875 INFO [StoreOpener-3c05f66f0b4334a1f317161bff1a01e6-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3c05f66f0b4334a1f317161bff1a01e6 columnFamilyName A 2024-11-18T20:14:33,876 DEBUG [StoreOpener-3c05f66f0b4334a1f317161bff1a01e6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:33,876 INFO [StoreOpener-3c05f66f0b4334a1f317161bff1a01e6-1 {}] regionserver.HStore(327): Store=3c05f66f0b4334a1f317161bff1a01e6/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T20:14:33,877 INFO [StoreOpener-3c05f66f0b4334a1f317161bff1a01e6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 3c05f66f0b4334a1f317161bff1a01e6 2024-11-18T20:14:33,878 INFO [StoreOpener-3c05f66f0b4334a1f317161bff1a01e6-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3c05f66f0b4334a1f317161bff1a01e6 columnFamilyName B 2024-11-18T20:14:33,878 DEBUG [StoreOpener-3c05f66f0b4334a1f317161bff1a01e6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:33,878 INFO [StoreOpener-3c05f66f0b4334a1f317161bff1a01e6-1 {}] regionserver.HStore(327): Store=3c05f66f0b4334a1f317161bff1a01e6/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T20:14:33,878 INFO [StoreOpener-3c05f66f0b4334a1f317161bff1a01e6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 3c05f66f0b4334a1f317161bff1a01e6 2024-11-18T20:14:33,879 INFO [StoreOpener-3c05f66f0b4334a1f317161bff1a01e6-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3c05f66f0b4334a1f317161bff1a01e6 columnFamilyName C 2024-11-18T20:14:33,879 DEBUG [StoreOpener-3c05f66f0b4334a1f317161bff1a01e6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:33,879 INFO [StoreOpener-3c05f66f0b4334a1f317161bff1a01e6-1 {}] regionserver.HStore(327): Store=3c05f66f0b4334a1f317161bff1a01e6/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T20:14:33,879 DEBUG [RS_OPEN_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] regionserver.HRegion(1038): replaying wal for 3c05f66f0b4334a1f317161bff1a01e6 2024-11-18T20:14:33,880 DEBUG [RS_OPEN_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6 2024-11-18T20:14:33,881 DEBUG [RS_OPEN_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6 2024-11-18T20:14:33,882 DEBUG [RS_OPEN_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] regionserver.HRegion(1048): stopping wal replay for 3c05f66f0b4334a1f317161bff1a01e6 2024-11-18T20:14:33,883 DEBUG [RS_OPEN_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] regionserver.HRegion(1060): Cleaning up temporary data for 3c05f66f0b4334a1f317161bff1a01e6 2024-11-18T20:14:33,883 DEBUG [RS_OPEN_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-18T20:14:33,885 DEBUG [RS_OPEN_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] regionserver.HRegion(1093): writing seq id for 3c05f66f0b4334a1f317161bff1a01e6 2024-11-18T20:14:33,886 INFO [RS_OPEN_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] regionserver.HRegion(1114): Opened 3c05f66f0b4334a1f317161bff1a01e6; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66047120, jitterRate=-0.015821218490600586}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-18T20:14:33,886 DEBUG [RS_OPEN_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 3c05f66f0b4334a1f317161bff1a01e6 2024-11-18T20:14:33,887 DEBUG [RS_OPEN_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] regionserver.HRegion(1006): Region open journal for 3c05f66f0b4334a1f317161bff1a01e6: Running coprocessor pre-open hook at 1731960873872Writing region info on filesystem at 1731960873872Initializing all the Stores at 1731960873873 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731960873873Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731960873874 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731960873874Cleaning up temporary data from old regions at 1731960873883 (+9 ms)Running coprocessor post-open hooks at 1731960873886 (+3 ms)Region opened successfully at 1731960873886 2024-11-18T20:14:33,887 INFO [RS_OPEN_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., pid=153, masterSystemTime=1731960873867 2024-11-18T20:14:33,890 DEBUG [RS_OPEN_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6. 2024-11-18T20:14:33,890 INFO [RS_OPEN_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6. 2024-11-18T20:14:33,890 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=151 updating hbase:meta row=3c05f66f0b4334a1f317161bff1a01e6, regionState=OPEN, openSeqNum=5, regionLocation=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:33,892 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=153, ppid=151, state=RUNNABLE, hasLock=false; OpenRegionProcedure 3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 because future has completed 2024-11-18T20:14:33,894 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=153, resume processing ppid=151 2024-11-18T20:14:33,894 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=153, ppid=151, state=SUCCESS, hasLock=false; OpenRegionProcedure 3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 in 181 msec 2024-11-18T20:14:33,896 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=151, resume processing ppid=150 2024-11-18T20:14:33,896 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=151, ppid=150, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=3c05f66f0b4334a1f317161bff1a01e6, REOPEN/MOVE in 511 msec 2024-11-18T20:14:33,898 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=150, resume processing ppid=149 2024-11-18T20:14:33,898 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=150, ppid=149, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 518 msec 2024-11-18T20:14:33,901 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=149, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 939 msec 2024-11-18T20:14:33,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=149 2024-11-18T20:14:33,902 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-18T20:14:33,903 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@185b2823, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:14:33,903 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c0a89b2656d4,41277,-1 for getting cluster id 2024-11-18T20:14:33,903 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T20:14:33,904 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ac99df6b-7791-4693-a81a-6a5a39504490' 2024-11-18T20:14:33,904 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T20:14:33,904 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ac99df6b-7791-4693-a81a-6a5a39504490" 2024-11-18T20:14:33,904 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6ce283e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:14:33,904 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c0a89b2656d4,41277,-1] 2024-11-18T20:14:33,904 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T20:14:33,905 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:14:33,905 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54472, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T20:14:33,906 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@76200c33, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:14:33,906 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@22faf5ac, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:14:33,906 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c0a89b2656d4,41277,-1 for getting cluster id 2024-11-18T20:14:33,906 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T20:14:33,907 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ac99df6b-7791-4693-a81a-6a5a39504490' 2024-11-18T20:14:33,907 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T20:14:33,907 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ac99df6b-7791-4693-a81a-6a5a39504490" 2024-11-18T20:14:33,907 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@328a5baf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:14:33,907 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c0a89b2656d4,41277,-1] 2024-11-18T20:14:33,907 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T20:14:33,908 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:14:33,908 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54480, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T20:14:33,909 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5d2649ac, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:14:33,909 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@744b0f71, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:14:33,909 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c0a89b2656d4,41277,-1 for getting cluster id 2024-11-18T20:14:33,909 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T20:14:33,910 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ac99df6b-7791-4693-a81a-6a5a39504490' 2024-11-18T20:14:33,910 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T20:14:33,910 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ac99df6b-7791-4693-a81a-6a5a39504490" 2024-11-18T20:14:33,910 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5cab1004, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:14:33,910 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c0a89b2656d4,41277,-1] 2024-11-18T20:14:33,910 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T20:14:33,910 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:14:33,911 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54500, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T20:14:33,911 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1040e258, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:14:33,912 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@143fb08, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:14:33,912 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c0a89b2656d4,41277,-1 for getting cluster id 2024-11-18T20:14:33,912 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T20:14:33,913 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ac99df6b-7791-4693-a81a-6a5a39504490' 2024-11-18T20:14:33,913 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T20:14:33,913 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ac99df6b-7791-4693-a81a-6a5a39504490" 2024-11-18T20:14:33,913 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4e0f9961, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:14:33,913 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c0a89b2656d4,41277,-1] 2024-11-18T20:14:33,913 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T20:14:33,913 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:14:33,914 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54508, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T20:14:33,914 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@366facf3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:14:33,915 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@40503a36, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:14:33,915 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c0a89b2656d4,41277,-1 for getting cluster id 2024-11-18T20:14:33,915 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T20:14:33,916 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ac99df6b-7791-4693-a81a-6a5a39504490' 2024-11-18T20:14:33,916 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T20:14:33,916 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ac99df6b-7791-4693-a81a-6a5a39504490" 2024-11-18T20:14:33,916 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@57360717, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:14:33,916 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c0a89b2656d4,41277,-1] 2024-11-18T20:14:33,916 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T20:14:33,917 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:14:33,917 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54528, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T20:14:33,918 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@54914313, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:14:33,919 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5d7db06e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:14:33,919 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c0a89b2656d4,41277,-1 for getting cluster id 2024-11-18T20:14:33,919 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T20:14:33,919 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ac99df6b-7791-4693-a81a-6a5a39504490' 2024-11-18T20:14:33,919 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T20:14:33,919 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ac99df6b-7791-4693-a81a-6a5a39504490" 2024-11-18T20:14:33,920 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@400f35ed, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:14:33,920 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c0a89b2656d4,41277,-1] 2024-11-18T20:14:33,920 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T20:14:33,920 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:14:33,920 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54544, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T20:14:33,921 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5e2b92e4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:14:33,921 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@763ca8cc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:14:33,922 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c0a89b2656d4,41277,-1 for getting cluster id 2024-11-18T20:14:33,922 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T20:14:33,922 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ac99df6b-7791-4693-a81a-6a5a39504490' 2024-11-18T20:14:33,922 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T20:14:33,923 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ac99df6b-7791-4693-a81a-6a5a39504490" 2024-11-18T20:14:33,923 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@70c84aa4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:14:33,923 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c0a89b2656d4,41277,-1] 2024-11-18T20:14:33,923 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T20:14:33,923 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:14:33,924 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54560, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T20:14:33,924 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@494bf4cf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:14:33,925 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7122db2b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:14:33,925 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c0a89b2656d4,41277,-1 for getting cluster id 2024-11-18T20:14:33,925 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T20:14:33,926 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ac99df6b-7791-4693-a81a-6a5a39504490' 2024-11-18T20:14:33,926 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T20:14:33,926 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ac99df6b-7791-4693-a81a-6a5a39504490" 2024-11-18T20:14:33,927 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@171e1fcc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:14:33,927 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c0a89b2656d4,41277,-1] 2024-11-18T20:14:33,927 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T20:14:33,927 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:14:33,928 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54574, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T20:14:33,928 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1efee4c4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:14:33,929 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3666b173, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:14:33,929 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c0a89b2656d4,41277,-1 for getting cluster id 2024-11-18T20:14:33,929 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T20:14:33,930 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ac99df6b-7791-4693-a81a-6a5a39504490' 2024-11-18T20:14:33,930 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T20:14:33,930 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ac99df6b-7791-4693-a81a-6a5a39504490" 2024-11-18T20:14:33,931 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3b410e68, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:14:33,931 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c0a89b2656d4,41277,-1] 2024-11-18T20:14:33,931 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T20:14:33,931 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:14:33,932 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54580, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T20:14:33,932 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1f055e28, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:14:33,933 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@77ff027c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:14:33,933 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c0a89b2656d4,41277,-1 for getting cluster id 2024-11-18T20:14:33,933 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T20:14:33,934 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ac99df6b-7791-4693-a81a-6a5a39504490' 2024-11-18T20:14:33,934 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T20:14:33,934 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ac99df6b-7791-4693-a81a-6a5a39504490" 2024-11-18T20:14:33,934 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2e9fc9e1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:14:33,934 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c0a89b2656d4,41277,-1] 2024-11-18T20:14:33,935 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T20:14:33,935 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:14:33,935 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54598, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T20:14:33,936 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@759317c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T20:14:33,937 DEBUG [Thread-2496 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T20:14:33,937 DEBUG [Thread-2511 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T20:14:33,937 DEBUG [Thread-2503 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T20:14:33,937 DEBUG [Thread-2500 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T20:14:33,937 DEBUG [Thread-2494 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T20:14:33,937 DEBUG [Thread-2492 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T20:14:33,937 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41277 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-18T20:14:33,937 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c0a89b2656d4,35963,1731960718223, seqNum=-1] 2024-11-18T20:14:33,937 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c0a89b2656d4,35963,1731960718223, seqNum=-1] 2024-11-18T20:14:33,937 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c0a89b2656d4,35963,1731960718223, seqNum=-1] 2024-11-18T20:14:33,937 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c0a89b2656d4,35963,1731960718223, seqNum=-1] 2024-11-18T20:14:33,938 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T20:14:33,938 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T20:14:33,938 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T20:14:33,938 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c0a89b2656d4,35963,1731960718223, seqNum=-1] 2024-11-18T20:14:33,938 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T20:14:33,938 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T20:14:33,938 DEBUG [Thread-2509 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T20:14:33,939 DEBUG [Thread-2505 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T20:14:33,939 DEBUG [Thread-2498 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T20:14:33,939 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51798, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T20:14:33,939 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51816, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T20:14:33,939 DEBUG [Thread-2507 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T20:14:33,939 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51796, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T20:14:33,939 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c0a89b2656d4,35963,1731960718223, seqNum=-1] 2024-11-18T20:14:33,939 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51830, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T20:14:33,939 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T20:14:33,939 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c0a89b2656d4,35963,1731960718223, seqNum=-1] 2024-11-18T20:14:33,939 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c0a89b2656d4,35963,1731960718223, seqNum=-1] 2024-11-18T20:14:33,939 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T20:14:33,939 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T20:14:33,940 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c0a89b2656d4,35963,1731960718223, seqNum=-1] 2024-11-18T20:14:33,940 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51828, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T20:14:33,940 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c0a89b2656d4,35963,1731960718223, seqNum=-1] 2024-11-18T20:14:33,940 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T20:14:33,940 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T20:14:33,940 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5] 2024-11-18T20:14:33,940 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51842, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T20:14:33,940 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51868, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T20:14:33,941 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51882, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T20:14:33,941 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5] 2024-11-18T20:14:33,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41277 {}] procedure2.ProcedureExecutor(1139): Stored pid=154, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees 2024-11-18T20:14:33,941 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51852, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T20:14:33,941 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51886, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T20:14:33,941 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5] 2024-11-18T20:14:33,942 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5] 2024-11-18T20:14:33,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-18T20:14:33,942 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5] 2024-11-18T20:14:33,943 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=154, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-18T20:14:33,943 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5] 2024-11-18T20:14:33,943 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5] 2024-11-18T20:14:33,943 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5] 2024-11-18T20:14:33,944 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5] 2024-11-18T20:14:33,944 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=154, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-18T20:14:33,944 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5] 2024-11-18T20:14:33,944 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=155, ppid=154, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-18T20:14:33,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(8855): Flush requested on 3c05f66f0b4334a1f317161bff1a01e6 2024-11-18T20:14:33,948 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-18T20:14:33,948 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3c05f66f0b4334a1f317161bff1a01e6 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-18T20:14:33,966 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:33,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51828 deadline: 1731960933963, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:33,967 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:33,967 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:33,967 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:14:33,967 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:33,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51886 deadline: 1731960933964, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:33,967 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:33,968 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:33,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51868 deadline: 1731960933965, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:33,968 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:33,968 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:14:33,968 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:33,968 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:33,968 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:14:33,968 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:33,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51830 deadline: 1731960933966, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:33,968 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:33,968 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:33,968 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:14:33,969 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:33,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51816 deadline: 1731960933966, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:33,969 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:33,969 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:33,969 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:14:33,974 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111899c5cf6640ad4075931c1029ea8feacd_3c05f66f0b4334a1f317161bff1a01e6 is 50, key is test_row_0/A:col10/1731960873944/Put/seqid=0 2024-11-18T20:14:33,979 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742386_1562 (size=17034) 2024-11-18T20:14:34,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-18T20:14:34,075 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:34,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51828 deadline: 1731960934073, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:34,075 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:34,075 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:34,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51886 deadline: 1731960934073, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:34,075 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:34,075 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:14:34,075 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:34,075 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:34,075 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:14:34,075 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:34,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51830 deadline: 1731960934073, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:34,076 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:34,076 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:34,076 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:14:34,076 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:34,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51816 deadline: 1731960934074, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:34,076 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:34,076 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:34,076 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:34,076 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:14:34,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51868 deadline: 1731960934074, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:34,076 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:34,076 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:34,076 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:14:34,097 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35963 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-18T20:14:34,097 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6. 2024-11-18T20:14:34,098 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6. as already flushing 2024-11-18T20:14:34,098 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6. 2024-11-18T20:14:34,098 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 3c05f66f0b4334a1f317161bff1a01e6, NAME => 'TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:14:34,098 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 3c05f66f0b4334a1f317161bff1a01e6, NAME => 'TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:14:34,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41277 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3c05f66f0b4334a1f317161bff1a01e6, NAME => 'TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3c05f66f0b4334a1f317161bff1a01e6, NAME => 'TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:14:34,250 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35963 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-18T20:14:34,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6. 2024-11-18T20:14:34,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6. as already flushing 2024-11-18T20:14:34,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6. 2024-11-18T20:14:34,250 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 3c05f66f0b4334a1f317161bff1a01e6, NAME => 'TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:14:34,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 3c05f66f0b4334a1f317161bff1a01e6, NAME => 'TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:14:34,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41277 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3c05f66f0b4334a1f317161bff1a01e6, NAME => 'TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3c05f66f0b4334a1f317161bff1a01e6, NAME => 'TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:14:34,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-18T20:14:34,284 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:34,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51828 deadline: 1731960934283, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:34,285 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:34,285 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:34,285 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:34,285 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:14:34,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51886 deadline: 1731960934283, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:34,285 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:34,285 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:34,285 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:14:34,285 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:34,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51830 deadline: 1731960934283, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:34,285 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:34,285 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:34,285 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:14:34,285 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:34,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51816 deadline: 1731960934284, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:34,286 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:34,286 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:34,286 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:14:34,286 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:34,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51868 deadline: 1731960934284, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:34,286 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:34,286 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:34,286 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:14:34,379 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:34,382 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111899c5cf6640ad4075931c1029ea8feacd_3c05f66f0b4334a1f317161bff1a01e6 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111899c5cf6640ad4075931c1029ea8feacd_3c05f66f0b4334a1f317161bff1a01e6 2024-11-18T20:14:34,382 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/.tmp/A/e3bf4d6373ad4fe4b376de9dbaee5f38, store: [table=TestAcidGuarantees family=A region=3c05f66f0b4334a1f317161bff1a01e6] 2024-11-18T20:14:34,382 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/.tmp/A/e3bf4d6373ad4fe4b376de9dbaee5f38 is 175, key is test_row_0/A:col10/1731960873944/Put/seqid=0 2024-11-18T20:14:34,385 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742387_1563 (size=48139) 2024-11-18T20:14:34,402 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35963 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-18T20:14:34,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6. 2024-11-18T20:14:34,403 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6. as already flushing 2024-11-18T20:14:34,403 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6. 2024-11-18T20:14:34,403 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 3c05f66f0b4334a1f317161bff1a01e6, NAME => 'TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:14:34,403 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 3c05f66f0b4334a1f317161bff1a01e6, NAME => 'TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:14:34,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41277 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3c05f66f0b4334a1f317161bff1a01e6, NAME => 'TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3c05f66f0b4334a1f317161bff1a01e6, NAME => 'TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:14:34,555 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35963 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-18T20:14:34,555 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6. 2024-11-18T20:14:34,555 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6. as already flushing 2024-11-18T20:14:34,555 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6. 2024-11-18T20:14:34,555 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 3c05f66f0b4334a1f317161bff1a01e6, NAME => 'TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:14:34,555 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 3c05f66f0b4334a1f317161bff1a01e6, NAME => 'TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:14:34,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41277 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3c05f66f0b4334a1f317161bff1a01e6, NAME => 'TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3c05f66f0b4334a1f317161bff1a01e6, NAME => 'TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:14:34,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-18T20:14:34,595 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:34,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51828 deadline: 1731960934593, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:34,595 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:34,595 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:34,595 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:34,595 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:14:34,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51886 deadline: 1731960934593, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:34,595 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:34,595 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:34,595 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:14:34,595 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:34,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51830 deadline: 1731960934593, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:34,596 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:34,596 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:34,596 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:14:34,596 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:34,596 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:34,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51816 deadline: 1731960934593, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:34,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51868 deadline: 1731960934593, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:34,596 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:34,596 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:34,596 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:34,596 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:34,596 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:14:34,596 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:14:34,707 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35963 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-18T20:14:34,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6. 2024-11-18T20:14:34,708 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6. as already flushing 2024-11-18T20:14:34,708 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6. 2024-11-18T20:14:34,708 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 3c05f66f0b4334a1f317161bff1a01e6, NAME => 'TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:14:34,708 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 3c05f66f0b4334a1f317161bff1a01e6, NAME => 'TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:14:34,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41277 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3c05f66f0b4334a1f317161bff1a01e6, NAME => 'TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3c05f66f0b4334a1f317161bff1a01e6, NAME => 'TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:14:34,786 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=16, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/.tmp/A/e3bf4d6373ad4fe4b376de9dbaee5f38 2024-11-18T20:14:34,804 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/.tmp/B/dde5d7090e37498ba6387e978b2b6051 is 50, key is test_row_0/B:col10/1731960873944/Put/seqid=0 2024-11-18T20:14:34,806 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742388_1564 (size=12001) 2024-11-18T20:14:34,807 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/.tmp/B/dde5d7090e37498ba6387e978b2b6051 2024-11-18T20:14:34,827 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/.tmp/C/b45e7b7087184f66b871753c54948e6f is 50, key is test_row_0/C:col10/1731960873944/Put/seqid=0 2024-11-18T20:14:34,834 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742389_1565 (size=12001) 2024-11-18T20:14:34,860 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35963 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-18T20:14:34,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6. 2024-11-18T20:14:34,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6. as already flushing 2024-11-18T20:14:34,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6. 2024-11-18T20:14:34,860 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 3c05f66f0b4334a1f317161bff1a01e6, NAME => 'TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:14:34,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 3c05f66f0b4334a1f317161bff1a01e6, NAME => 'TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:14:34,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41277 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3c05f66f0b4334a1f317161bff1a01e6, NAME => 'TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3c05f66f0b4334a1f317161bff1a01e6, NAME => 'TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:14:35,013 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35963 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-18T20:14:35,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6. 2024-11-18T20:14:35,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6. as already flushing 2024-11-18T20:14:35,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6. 2024-11-18T20:14:35,013 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 3c05f66f0b4334a1f317161bff1a01e6, NAME => 'TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:14:35,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 3c05f66f0b4334a1f317161bff1a01e6, NAME => 'TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:14:35,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41277 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3c05f66f0b4334a1f317161bff1a01e6, NAME => 'TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3c05f66f0b4334a1f317161bff1a01e6, NAME => 'TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:14:35,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-18T20:14:35,104 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:35,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51828 deadline: 1731960935103, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:35,105 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:35,105 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:35,105 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:35,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51830 deadline: 1731960935103, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:35,105 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:14:35,105 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:35,105 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:35,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51816 deadline: 1731960935103, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:35,105 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:35,105 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:14:35,105 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:35,105 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:35,105 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:35,105 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:14:35,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51886 deadline: 1731960935103, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:35,106 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:35,106 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:35,106 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:35,106 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:14:35,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51868 deadline: 1731960935103, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:35,106 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:35,106 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:35,106 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:14:35,165 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35963 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-18T20:14:35,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6. 2024-11-18T20:14:35,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6. as already flushing 2024-11-18T20:14:35,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6. 2024-11-18T20:14:35,166 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 3c05f66f0b4334a1f317161bff1a01e6, NAME => 'TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:14:35,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 3c05f66f0b4334a1f317161bff1a01e6, NAME => 'TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:14:35,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41277 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3c05f66f0b4334a1f317161bff1a01e6, NAME => 'TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3c05f66f0b4334a1f317161bff1a01e6, NAME => 'TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:14:35,235 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/.tmp/C/b45e7b7087184f66b871753c54948e6f 2024-11-18T20:14:35,238 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/.tmp/A/e3bf4d6373ad4fe4b376de9dbaee5f38 as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/A/e3bf4d6373ad4fe4b376de9dbaee5f38 2024-11-18T20:14:35,240 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/A/e3bf4d6373ad4fe4b376de9dbaee5f38, entries=250, sequenceid=16, filesize=47.0 K 2024-11-18T20:14:35,241 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/.tmp/B/dde5d7090e37498ba6387e978b2b6051 as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/B/dde5d7090e37498ba6387e978b2b6051 2024-11-18T20:14:35,244 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/B/dde5d7090e37498ba6387e978b2b6051, entries=150, sequenceid=16, filesize=11.7 K 2024-11-18T20:14:35,244 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/.tmp/C/b45e7b7087184f66b871753c54948e6f as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/C/b45e7b7087184f66b871753c54948e6f 2024-11-18T20:14:35,247 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/C/b45e7b7087184f66b871753c54948e6f, entries=150, sequenceid=16, filesize=11.7 K 2024-11-18T20:14:35,247 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 3c05f66f0b4334a1f317161bff1a01e6 in 1299ms, sequenceid=16, compaction requested=false 2024-11-18T20:14:35,247 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-18T20:14:35,248 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3c05f66f0b4334a1f317161bff1a01e6: 2024-11-18T20:14:35,318 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35963 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-18T20:14:35,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6. 2024-11-18T20:14:35,318 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2902): Flushing 3c05f66f0b4334a1f317161bff1a01e6 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-18T20:14:35,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111892e9dea7cc61452e950d628ab3abb045_3c05f66f0b4334a1f317161bff1a01e6 is 50, key is test_row_0/A:col10/1731960873965/Put/seqid=0 2024-11-18T20:14:35,327 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742390_1566 (size=12154) 2024-11-18T20:14:35,328 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:35,332 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111892e9dea7cc61452e950d628ab3abb045_3c05f66f0b4334a1f317161bff1a01e6 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111892e9dea7cc61452e950d628ab3abb045_3c05f66f0b4334a1f317161bff1a01e6 2024-11-18T20:14:35,333 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/.tmp/A/dcd8f8aaff52436385c61886be3a3dc3, store: [table=TestAcidGuarantees family=A region=3c05f66f0b4334a1f317161bff1a01e6] 2024-11-18T20:14:35,333 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/.tmp/A/dcd8f8aaff52436385c61886be3a3dc3 is 175, key is test_row_0/A:col10/1731960873965/Put/seqid=0 2024-11-18T20:14:35,337 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742391_1567 (size=30955) 2024-11-18T20:14:35,578 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-18T20:14:35,737 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=40, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/.tmp/A/dcd8f8aaff52436385c61886be3a3dc3 2024-11-18T20:14:35,755 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/.tmp/B/92f9e0980b0e4e2ca402204d5b4f7c8e is 50, key is test_row_0/B:col10/1731960873965/Put/seqid=0 2024-11-18T20:14:35,757 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742392_1568 (size=12001) 2024-11-18T20:14:36,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-18T20:14:36,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(8855): Flush requested on 3c05f66f0b4334a1f317161bff1a01e6 2024-11-18T20:14:36,115 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6. as already flushing 2024-11-18T20:14:36,127 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:36,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51828 deadline: 1731960936123, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:36,128 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:36,128 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:36,128 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:14:36,128 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:36,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51830 deadline: 1731960936125, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:36,129 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:36,129 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:36,129 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:14:36,131 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:36,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51816 deadline: 1731960936127, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:36,131 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:36,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51886 deadline: 1731960936128, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:36,132 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:36,132 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:36,132 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:36,132 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:14:36,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51868 deadline: 1731960936128, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:36,132 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:36,132 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:36,132 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:14:36,132 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:36,132 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:36,132 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:14:36,158 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=40 (bloomFilter=true), to=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/.tmp/B/92f9e0980b0e4e2ca402204d5b4f7c8e 2024-11-18T20:14:36,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/.tmp/C/b308f86f673b4f4c911dceb4b2e69145 is 50, key is test_row_0/C:col10/1731960873965/Put/seqid=0 2024-11-18T20:14:36,166 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742393_1569 (size=12001) 2024-11-18T20:14:36,234 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:36,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51828 deadline: 1731960936233, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:36,235 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:36,235 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:36,235 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:36,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51830 deadline: 1731960936233, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:36,235 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:14:36,235 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:36,235 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:36,235 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:14:36,244 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:36,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51816 deadline: 1731960936242, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:36,244 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:36,244 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:36,244 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:36,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51886 deadline: 1731960936243, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:36,244 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:14:36,244 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:36,244 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:36,244 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:36,244 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:14:36,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51868 deadline: 1731960936243, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:36,245 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:36,245 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:36,245 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:14:36,445 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:36,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51828 deadline: 1731960936443, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:36,445 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:36,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51830 deadline: 1731960936443, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:36,445 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:36,445 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:36,445 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:14:36,445 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:36,445 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:36,445 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:14:36,454 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:36,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51816 deadline: 1731960936453, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:36,454 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:36,454 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:36,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51886 deadline: 1731960936453, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:36,455 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:36,455 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:14:36,455 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:36,455 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:36,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51868 deadline: 1731960936453, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:36,455 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:36,455 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:14:36,455 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:36,455 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:36,455 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:14:36,567 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=40 (bloomFilter=true), to=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/.tmp/C/b308f86f673b4f4c911dceb4b2e69145 2024-11-18T20:14:36,570 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/.tmp/A/dcd8f8aaff52436385c61886be3a3dc3 as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/A/dcd8f8aaff52436385c61886be3a3dc3 2024-11-18T20:14:36,573 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/A/dcd8f8aaff52436385c61886be3a3dc3, entries=150, sequenceid=40, filesize=30.2 K 2024-11-18T20:14:36,573 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/.tmp/B/92f9e0980b0e4e2ca402204d5b4f7c8e as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/B/92f9e0980b0e4e2ca402204d5b4f7c8e 2024-11-18T20:14:36,578 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/B/92f9e0980b0e4e2ca402204d5b4f7c8e, entries=150, sequenceid=40, filesize=11.7 K 2024-11-18T20:14:36,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/.tmp/C/b308f86f673b4f4c911dceb4b2e69145 as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/C/b308f86f673b4f4c911dceb4b2e69145 2024-11-18T20:14:36,582 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/C/b308f86f673b4f4c911dceb4b2e69145, entries=150, sequenceid=40, filesize=11.7 K 2024-11-18T20:14:36,582 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=67.09 KB/68700 for 3c05f66f0b4334a1f317161bff1a01e6 in 1264ms, sequenceid=40, compaction requested=false 2024-11-18T20:14:36,583 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2603): Flush status journal for 3c05f66f0b4334a1f317161bff1a01e6: 2024-11-18T20:14:36,583 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6. 2024-11-18T20:14:36,583 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=155 2024-11-18T20:14:36,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41277 {}] master.HMaster(4169): Remote procedure done, pid=155 2024-11-18T20:14:36,585 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=155, resume processing ppid=154 2024-11-18T20:14:36,585 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=155, ppid=154, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.6400 sec 2024-11-18T20:14:36,586 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=154, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees in 2.6480 sec 2024-11-18T20:14:36,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(8855): Flush requested on 3c05f66f0b4334a1f317161bff1a01e6 2024-11-18T20:14:36,755 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-18T20:14:36,755 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3c05f66f0b4334a1f317161bff1a01e6 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-11-18T20:14:36,761 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118b77fc8366aca4ea5a782f9c5eebd0069_3c05f66f0b4334a1f317161bff1a01e6 is 50, key is test_row_0/A:col10/1731960876127/Put/seqid=0 2024-11-18T20:14:36,765 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742394_1570 (size=12154) 2024-11-18T20:14:36,807 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:36,807 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:36,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51828 deadline: 1731960936802, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:36,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51868 deadline: 1731960936801, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:36,807 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:36,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51830 deadline: 1731960936803, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:36,807 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:36,807 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:36,807 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:36,807 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:36,807 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:14:36,807 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:14:36,807 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:36,807 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:36,807 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:14:36,810 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:36,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51816 deadline: 1731960936806, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:36,810 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:36,810 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:36,811 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:36,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51886 deadline: 1731960936807, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:36,811 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:14:36,811 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:36,811 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:36,811 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:14:36,914 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:36,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51868 deadline: 1731960936913, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:36,914 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:36,914 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:36,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51828 deadline: 1731960936913, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:36,914 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:36,914 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:14:36,914 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:36,914 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:36,914 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:36,914 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:14:36,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51830 deadline: 1731960936913, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:36,915 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:36,915 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:36,915 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:36,915 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:14:36,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51816 deadline: 1731960936913, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:36,915 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:36,915 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:36,915 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:36,915 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:14:36,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51886 deadline: 1731960936913, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:36,915 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:36,915 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:36,915 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:14:37,124 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:37,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51868 deadline: 1731960937123, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:37,124 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:37,125 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:37,125 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:37,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51828 deadline: 1731960937123, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:37,125 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:14:37,125 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:37,125 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:37,125 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:37,125 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:14:37,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51830 deadline: 1731960937123, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:37,125 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:37,125 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:37,125 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:37,125 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:14:37,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51816 deadline: 1731960937123, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:37,125 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:37,125 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:37,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51886 deadline: 1731960937123, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:37,125 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:37,125 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:14:37,126 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:37,126 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:37,126 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:14:37,165 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:37,168 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118b77fc8366aca4ea5a782f9c5eebd0069_3c05f66f0b4334a1f317161bff1a01e6 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118b77fc8366aca4ea5a782f9c5eebd0069_3c05f66f0b4334a1f317161bff1a01e6 2024-11-18T20:14:37,168 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/.tmp/A/825d5734db984c7aa6293f8eba885cd5, store: [table=TestAcidGuarantees family=A region=3c05f66f0b4334a1f317161bff1a01e6] 2024-11-18T20:14:37,169 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/.tmp/A/825d5734db984c7aa6293f8eba885cd5 is 175, key is test_row_0/A:col10/1731960876127/Put/seqid=0 2024-11-18T20:14:37,171 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742395_1571 (size=30955) 2024-11-18T20:14:37,435 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:37,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51868 deadline: 1731960937433, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:37,435 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:37,435 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:37,435 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:37,435 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:14:37,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51828 deadline: 1731960937433, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:37,435 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:37,435 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:37,435 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:37,435 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:14:37,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51830 deadline: 1731960937433, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:37,436 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:37,436 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:37,436 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:37,436 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:14:37,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51816 deadline: 1731960937433, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:37,436 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:37,436 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:37,436 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:37,436 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:14:37,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51886 deadline: 1731960937433, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:37,436 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:37,436 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:37,436 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:14:37,572 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=55, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/.tmp/A/825d5734db984c7aa6293f8eba885cd5 2024-11-18T20:14:37,578 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/.tmp/B/1ac398a88c55448cabc84eaf0e959165 is 50, key is test_row_0/B:col10/1731960876127/Put/seqid=0 2024-11-18T20:14:37,580 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742396_1572 (size=12001) 2024-11-18T20:14:37,945 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:37,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51868 deadline: 1731960937943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:37,945 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:37,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51828 deadline: 1731960937943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:37,945 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:37,945 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:37,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51830 deadline: 1731960937943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:37,946 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:37,946 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:14:37,946 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:37,946 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:37,946 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:37,946 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:14:37,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51816 deadline: 1731960937943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:37,946 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:37,946 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:37,946 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:37,946 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:14:37,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51886 deadline: 1731960937943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:37,946 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:37,946 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:37,946 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:14:37,949 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:37,949 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:37,949 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:14:37,981 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=55 (bloomFilter=true), to=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/.tmp/B/1ac398a88c55448cabc84eaf0e959165 2024-11-18T20:14:37,986 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/.tmp/C/6555f5af1fd54e3a93288abcfe896342 is 50, key is test_row_0/C:col10/1731960876127/Put/seqid=0 2024-11-18T20:14:37,990 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742397_1573 (size=12001) 2024-11-18T20:14:38,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-18T20:14:38,103 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-18T20:14:38,103 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41277 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-18T20:14:38,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41277 {}] procedure2.ProcedureExecutor(1139): Stored pid=156, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees 2024-11-18T20:14:38,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-18T20:14:38,105 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=156, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-18T20:14:38,105 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=156, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-18T20:14:38,105 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=157, ppid=156, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-18T20:14:38,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-18T20:14:38,258 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35963 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-18T20:14:38,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6. 2024-11-18T20:14:38,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6. as already flushing 2024-11-18T20:14:38,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6. 2024-11-18T20:14:38,258 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => 3c05f66f0b4334a1f317161bff1a01e6, NAME => 'TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:14:38,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => 3c05f66f0b4334a1f317161bff1a01e6, NAME => 'TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:14:38,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41277 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3c05f66f0b4334a1f317161bff1a01e6, NAME => 'TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3c05f66f0b4334a1f317161bff1a01e6, NAME => 'TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:14:38,391 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=55 (bloomFilter=true), to=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/.tmp/C/6555f5af1fd54e3a93288abcfe896342 2024-11-18T20:14:38,394 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/.tmp/A/825d5734db984c7aa6293f8eba885cd5 as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/A/825d5734db984c7aa6293f8eba885cd5 2024-11-18T20:14:38,396 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/A/825d5734db984c7aa6293f8eba885cd5, entries=150, sequenceid=55, filesize=30.2 K 2024-11-18T20:14:38,397 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/.tmp/B/1ac398a88c55448cabc84eaf0e959165 as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/B/1ac398a88c55448cabc84eaf0e959165 2024-11-18T20:14:38,399 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/B/1ac398a88c55448cabc84eaf0e959165, entries=150, sequenceid=55, filesize=11.7 K 2024-11-18T20:14:38,400 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/.tmp/C/6555f5af1fd54e3a93288abcfe896342 as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/C/6555f5af1fd54e3a93288abcfe896342 2024-11-18T20:14:38,402 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/C/6555f5af1fd54e3a93288abcfe896342, entries=150, sequenceid=55, filesize=11.7 K 2024-11-18T20:14:38,403 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=120.76 KB/123660 for 3c05f66f0b4334a1f317161bff1a01e6 in 1648ms, sequenceid=55, compaction requested=true 2024-11-18T20:14:38,403 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3c05f66f0b4334a1f317161bff1a01e6: 2024-11-18T20:14:38,403 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3c05f66f0b4334a1f317161bff1a01e6:A, priority=-2147483648, current under compaction store size is 1 2024-11-18T20:14:38,403 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T20:14:38,403 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:14:38,403 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3c05f66f0b4334a1f317161bff1a01e6:B, priority=-2147483648, current under compaction store size is 2 2024-11-18T20:14:38,403 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:14:38,403 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T20:14:38,403 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3c05f66f0b4334a1f317161bff1a01e6:C, priority=-2147483648, current under compaction store size is 3 2024-11-18T20:14:38,403 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T20:14:38,407 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T20:14:38,407 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 110049 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T20:14:38,407 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HStore(1541): 3c05f66f0b4334a1f317161bff1a01e6/B is initiating minor compaction (all files) 2024-11-18T20:14:38,407 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HStore(1541): 3c05f66f0b4334a1f317161bff1a01e6/A is initiating minor compaction (all files) 2024-11-18T20:14:38,407 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3c05f66f0b4334a1f317161bff1a01e6/B in TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6. 2024-11-18T20:14:38,407 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3c05f66f0b4334a1f317161bff1a01e6/A in TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6. 2024-11-18T20:14:38,407 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/B/dde5d7090e37498ba6387e978b2b6051, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/B/92f9e0980b0e4e2ca402204d5b4f7c8e, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/B/1ac398a88c55448cabc84eaf0e959165] into tmpdir=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/.tmp, totalSize=35.2 K 2024-11-18T20:14:38,407 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/A/e3bf4d6373ad4fe4b376de9dbaee5f38, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/A/dcd8f8aaff52436385c61886be3a3dc3, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/A/825d5734db984c7aa6293f8eba885cd5] into tmpdir=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/.tmp, totalSize=107.5 K 2024-11-18T20:14:38,407 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6. 2024-11-18T20:14:38,407 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6. files: [hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/A/e3bf4d6373ad4fe4b376de9dbaee5f38, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/A/dcd8f8aaff52436385c61886be3a3dc3, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/A/825d5734db984c7aa6293f8eba885cd5] 2024-11-18T20:14:38,407 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.Compactor(225): Compacting dde5d7090e37498ba6387e978b2b6051, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1731960873944 2024-11-18T20:14:38,407 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] compactions.Compactor(225): Compacting e3bf4d6373ad4fe4b376de9dbaee5f38, keycount=250, bloomtype=ROW, size=47.0 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1731960873944 2024-11-18T20:14:38,407 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.Compactor(225): Compacting 92f9e0980b0e4e2ca402204d5b4f7c8e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1731960873960 2024-11-18T20:14:38,407 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.Compactor(225): Compacting 1ac398a88c55448cabc84eaf0e959165, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=55, earliestPutTs=1731960876118 2024-11-18T20:14:38,407 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] compactions.Compactor(225): Compacting dcd8f8aaff52436385c61886be3a3dc3, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1731960873960 2024-11-18T20:14:38,408 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] compactions.Compactor(225): Compacting 825d5734db984c7aa6293f8eba885cd5, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=55, earliestPutTs=1731960876118 2024-11-18T20:14:38,410 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35963 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-18T20:14:38,410 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6. 2024-11-18T20:14:38,411 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2902): Flushing 3c05f66f0b4334a1f317161bff1a01e6 3/3 column families, dataSize=120.76 KB heapSize=317.16 KB 2024-11-18T20:14:38,411 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=3c05f66f0b4334a1f317161bff1a01e6] 2024-11-18T20:14:38,412 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3c05f66f0b4334a1f317161bff1a01e6#B#compaction#492 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:14:38,413 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/.tmp/B/8c89c95908234d6d84f6bf1f371176a0 is 50, key is test_row_0/B:col10/1731960876127/Put/seqid=0 2024-11-18T20:14:38,414 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241118caf22409eed942fda6561c3400331d22_3c05f66f0b4334a1f317161bff1a01e6 store=[table=TestAcidGuarantees family=A region=3c05f66f0b4334a1f317161bff1a01e6] 2024-11-18T20:14:38,416 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241118caf22409eed942fda6561c3400331d22_3c05f66f0b4334a1f317161bff1a01e6, store=[table=TestAcidGuarantees family=A region=3c05f66f0b4334a1f317161bff1a01e6] 2024-11-18T20:14:38,416 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118caf22409eed942fda6561c3400331d22_3c05f66f0b4334a1f317161bff1a01e6 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=3c05f66f0b4334a1f317161bff1a01e6] 2024-11-18T20:14:38,422 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742398_1574 (size=12104) 2024-11-18T20:14:38,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-18T20:14:38,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118d8d21a47b48a4fb2884f76852e1dcec3_3c05f66f0b4334a1f317161bff1a01e6 is 50, key is test_row_0/A:col10/1731960876772/Put/seqid=0 2024-11-18T20:14:38,427 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742399_1575 (size=12154) 2024-11-18T20:14:38,438 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742400_1576 (size=4469) 2024-11-18T20:14:38,439 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3c05f66f0b4334a1f317161bff1a01e6#A#compaction#491 average throughput is 0.87 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:14:38,439 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/.tmp/A/c5cd93c24e68438495fb8ff6b04e83b5 is 175, key is test_row_0/A:col10/1731960876127/Put/seqid=0 2024-11-18T20:14:38,451 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742401_1577 (size=31058) 2024-11-18T20:14:38,454 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/.tmp/A/c5cd93c24e68438495fb8ff6b04e83b5 as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/A/c5cd93c24e68438495fb8ff6b04e83b5 2024-11-18T20:14:38,458 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3c05f66f0b4334a1f317161bff1a01e6/A of 3c05f66f0b4334a1f317161bff1a01e6 into c5cd93c24e68438495fb8ff6b04e83b5(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:14:38,458 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3c05f66f0b4334a1f317161bff1a01e6: 2024-11-18T20:14:38,458 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., storeName=3c05f66f0b4334a1f317161bff1a01e6/A, priority=13, startTime=1731960878403; duration=0sec 2024-11-18T20:14:38,458 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T20:14:38,458 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3c05f66f0b4334a1f317161bff1a01e6:A 2024-11-18T20:14:38,458 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T20:14:38,459 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T20:14:38,459 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HStore(1541): 3c05f66f0b4334a1f317161bff1a01e6/C is initiating minor compaction (all files) 2024-11-18T20:14:38,459 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3c05f66f0b4334a1f317161bff1a01e6/C in TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6. 2024-11-18T20:14:38,459 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/C/b45e7b7087184f66b871753c54948e6f, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/C/b308f86f673b4f4c911dceb4b2e69145, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/C/6555f5af1fd54e3a93288abcfe896342] into tmpdir=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/.tmp, totalSize=35.2 K 2024-11-18T20:14:38,460 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] compactions.Compactor(225): Compacting b45e7b7087184f66b871753c54948e6f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1731960873944 2024-11-18T20:14:38,460 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] compactions.Compactor(225): Compacting b308f86f673b4f4c911dceb4b2e69145, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1731960873960 2024-11-18T20:14:38,460 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6555f5af1fd54e3a93288abcfe896342, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=55, earliestPutTs=1731960876118 2024-11-18T20:14:38,466 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3c05f66f0b4334a1f317161bff1a01e6#C#compaction#494 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:14:38,466 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/.tmp/C/fb61bacd061b4933b2c51aec4c23222e is 50, key is test_row_0/C:col10/1731960876127/Put/seqid=0 2024-11-18T20:14:38,469 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742402_1578 (size=12104) 2024-11-18T20:14:38,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-18T20:14:38,827 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/.tmp/B/8c89c95908234d6d84f6bf1f371176a0 as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/B/8c89c95908234d6d84f6bf1f371176a0 2024-11-18T20:14:38,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:38,830 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118d8d21a47b48a4fb2884f76852e1dcec3_3c05f66f0b4334a1f317161bff1a01e6 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118d8d21a47b48a4fb2884f76852e1dcec3_3c05f66f0b4334a1f317161bff1a01e6 2024-11-18T20:14:38,831 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/.tmp/A/c0cf7c7af19743ee921a3326887d970e, store: [table=TestAcidGuarantees family=A region=3c05f66f0b4334a1f317161bff1a01e6] 2024-11-18T20:14:38,831 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3c05f66f0b4334a1f317161bff1a01e6/B of 3c05f66f0b4334a1f317161bff1a01e6 into 8c89c95908234d6d84f6bf1f371176a0(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:14:38,831 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3c05f66f0b4334a1f317161bff1a01e6: 2024-11-18T20:14:38,831 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., storeName=3c05f66f0b4334a1f317161bff1a01e6/B, priority=13, startTime=1731960878403; duration=0sec 2024-11-18T20:14:38,831 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:14:38,831 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3c05f66f0b4334a1f317161bff1a01e6:B 2024-11-18T20:14:38,831 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/.tmp/A/c0cf7c7af19743ee921a3326887d970e is 175, key is test_row_0/A:col10/1731960876772/Put/seqid=0 2024-11-18T20:14:38,837 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742403_1579 (size=30955) 2024-11-18T20:14:38,873 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/.tmp/C/fb61bacd061b4933b2c51aec4c23222e as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/C/fb61bacd061b4933b2c51aec4c23222e 2024-11-18T20:14:38,876 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3c05f66f0b4334a1f317161bff1a01e6/C of 3c05f66f0b4334a1f317161bff1a01e6 into fb61bacd061b4933b2c51aec4c23222e(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:14:38,877 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3c05f66f0b4334a1f317161bff1a01e6: 2024-11-18T20:14:38,877 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., storeName=3c05f66f0b4334a1f317161bff1a01e6/C, priority=13, startTime=1731960878403; duration=0sec 2024-11-18T20:14:38,877 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:14:38,877 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3c05f66f0b4334a1f317161bff1a01e6:C 2024-11-18T20:14:38,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(8855): Flush requested on 3c05f66f0b4334a1f317161bff1a01e6 2024-11-18T20:14:38,955 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6. as already flushing 2024-11-18T20:14:38,975 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:38,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51816 deadline: 1731960938969, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:38,975 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:38,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51828 deadline: 1731960938972, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:38,975 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:38,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51830 deadline: 1731960938973, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:38,976 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:38,976 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:38,976 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:14:38,976 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:38,976 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:38,976 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:14:38,976 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:38,976 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:38,976 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:14:38,979 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:38,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51886 deadline: 1731960938974, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:38,979 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:38,979 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:38,979 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:14:38,979 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:38,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51868 deadline: 1731960938976, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:38,980 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:38,980 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:38,980 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:14:39,085 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:39,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51830 deadline: 1731960939083, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:39,085 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:39,085 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:39,085 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:39,085 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:14:39,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51828 deadline: 1731960939083, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:39,086 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:39,086 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:39,086 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:39,086 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:39,086 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:14:39,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51868 deadline: 1731960939083, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:39,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51886 deadline: 1731960939083, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:39,086 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:39,086 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:39,086 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:39,086 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:39,086 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:14:39,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51816 deadline: 1731960939083, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:39,086 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:39,086 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:14:39,086 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:39,086 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:39,086 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:14:39,238 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=76, memsize=40.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/.tmp/A/c0cf7c7af19743ee921a3326887d970e 2024-11-18T20:14:39,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-18T20:14:39,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/.tmp/B/a9969320227b4298a623918c098a470f is 50, key is test_row_0/B:col10/1731960876772/Put/seqid=0 2024-11-18T20:14:39,247 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742404_1580 (size=12001) 2024-11-18T20:14:39,295 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:39,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51830 deadline: 1731960939293, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:39,295 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:39,296 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:39,296 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:39,296 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:39,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51828 deadline: 1731960939293, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:39,296 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:14:39,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51868 deadline: 1731960939293, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:39,296 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:39,296 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:39,296 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:39,296 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:39,296 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:39,296 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:39,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51886 deadline: 1731960939293, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:39,296 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:14:39,296 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:14:39,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51816 deadline: 1731960939293, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:39,296 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:39,296 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:39,296 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:14:39,296 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:39,296 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:39,296 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:14:39,606 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:39,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51830 deadline: 1731960939603, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:39,606 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:39,606 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:39,606 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:39,606 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:14:39,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51828 deadline: 1731960939603, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:39,606 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:39,606 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:39,606 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:39,606 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:14:39,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51886 deadline: 1731960939604, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:39,607 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:39,607 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:39,607 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:39,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51868 deadline: 1731960939604, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:39,607 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:14:39,607 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:39,607 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:39,607 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:39,607 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:14:39,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51816 deadline: 1731960939604, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:39,607 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:39,607 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:39,607 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:14:39,648 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=76 (bloomFilter=true), to=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/.tmp/B/a9969320227b4298a623918c098a470f 2024-11-18T20:14:39,660 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/.tmp/C/076e25d583474d2ab57a817a1aeff2d2 is 50, key is test_row_0/C:col10/1731960876772/Put/seqid=0 2024-11-18T20:14:39,663 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742405_1581 (size=12001) 2024-11-18T20:14:39,664 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=76 (bloomFilter=true), to=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/.tmp/C/076e25d583474d2ab57a817a1aeff2d2 2024-11-18T20:14:39,678 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/.tmp/A/c0cf7c7af19743ee921a3326887d970e as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/A/c0cf7c7af19743ee921a3326887d970e 2024-11-18T20:14:39,681 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/A/c0cf7c7af19743ee921a3326887d970e, entries=150, sequenceid=76, filesize=30.2 K 2024-11-18T20:14:39,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/.tmp/B/a9969320227b4298a623918c098a470f as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/B/a9969320227b4298a623918c098a470f 2024-11-18T20:14:39,685 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/B/a9969320227b4298a623918c098a470f, entries=150, sequenceid=76, filesize=11.7 K 2024-11-18T20:14:39,686 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/.tmp/C/076e25d583474d2ab57a817a1aeff2d2 as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/C/076e25d583474d2ab57a817a1aeff2d2 2024-11-18T20:14:39,689 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/C/076e25d583474d2ab57a817a1aeff2d2, entries=150, sequenceid=76, filesize=11.7 K 2024-11-18T20:14:39,690 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~317.11 KB/324720, currentSize=80.51 KB/82440 for 3c05f66f0b4334a1f317161bff1a01e6 in 1279ms, sequenceid=76, compaction requested=false 2024-11-18T20:14:39,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2603): Flush status journal for 3c05f66f0b4334a1f317161bff1a01e6: 2024-11-18T20:14:39,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6. 2024-11-18T20:14:39,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=157 2024-11-18T20:14:39,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41277 {}] master.HMaster(4169): Remote procedure done, pid=157 2024-11-18T20:14:39,692 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=157, resume processing ppid=156 2024-11-18T20:14:39,692 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=157, ppid=156, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5860 sec 2024-11-18T20:14:39,694 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=156, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees in 1.5900 sec 2024-11-18T20:14:40,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(8855): Flush requested on 3c05f66f0b4334a1f317161bff1a01e6 2024-11-18T20:14:40,116 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-18T20:14:40,116 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3c05f66f0b4334a1f317161bff1a01e6 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2024-11-18T20:14:40,122 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411187fba2659cef243a2ac0f51b9c073d8e2_3c05f66f0b4334a1f317161bff1a01e6 is 50, key is test_row_0/A:col10/1731960880115/Put/seqid=0 2024-11-18T20:14:40,125 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742406_1582 (size=14594) 2024-11-18T20:14:40,129 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:40,131 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411187fba2659cef243a2ac0f51b9c073d8e2_3c05f66f0b4334a1f317161bff1a01e6 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411187fba2659cef243a2ac0f51b9c073d8e2_3c05f66f0b4334a1f317161bff1a01e6 2024-11-18T20:14:40,132 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/.tmp/A/bac0ca3c4b4f491cbdcab34ab03fec7d, store: [table=TestAcidGuarantees family=A region=3c05f66f0b4334a1f317161bff1a01e6] 2024-11-18T20:14:40,132 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/.tmp/A/bac0ca3c4b4f491cbdcab34ab03fec7d is 175, key is test_row_0/A:col10/1731960880115/Put/seqid=0 2024-11-18T20:14:40,139 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742407_1583 (size=39549) 2024-11-18T20:14:40,167 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:40,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51816 deadline: 1731960940161, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:40,167 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:40,167 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:40,167 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:14:40,171 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:40,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51828 deadline: 1731960940165, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:40,171 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:40,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51868 deadline: 1731960940166, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:40,171 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:40,171 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:40,171 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:14:40,171 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:40,171 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:40,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51830 deadline: 1731960940167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:40,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51886 deadline: 1731960940167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:40,172 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:40,172 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:40,172 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:14:40,172 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:40,172 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:40,172 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:14:40,172 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:40,172 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:40,172 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:14:40,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-18T20:14:40,253 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-18T20:14:40,253 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41277 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-18T20:14:40,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41277 {}] procedure2.ProcedureExecutor(1139): Stored pid=158, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees 2024-11-18T20:14:40,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-18T20:14:40,255 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=158, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-18T20:14:40,255 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=158, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-18T20:14:40,255 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=159, ppid=158, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-18T20:14:40,274 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:40,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51816 deadline: 1731960940273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:40,274 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:40,275 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:40,275 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:40,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51828 deadline: 1731960940273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:40,275 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:14:40,275 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:40,275 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:40,275 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:14:40,286 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:40,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51830 deadline: 1731960940283, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:40,287 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:40,287 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:40,287 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:40,287 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:14:40,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51886 deadline: 1731960940283, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:40,287 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:40,287 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:40,287 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:40,287 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:14:40,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51868 deadline: 1731960940283, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:40,287 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:40,287 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:40,287 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:14:40,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-18T20:14:40,407 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35963 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-18T20:14:40,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6. 2024-11-18T20:14:40,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6. as already flushing 2024-11-18T20:14:40,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6. 2024-11-18T20:14:40,407 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => 3c05f66f0b4334a1f317161bff1a01e6, NAME => 'TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:14:40,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => 3c05f66f0b4334a1f317161bff1a01e6, NAME => 'TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:14:40,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41277 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3c05f66f0b4334a1f317161bff1a01e6, NAME => 'TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3c05f66f0b4334a1f317161bff1a01e6, NAME => 'TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:14:40,486 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:40,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51816 deadline: 1731960940483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:40,487 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:40,487 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:40,487 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:40,487 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:14:40,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51828 deadline: 1731960940483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:40,487 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:40,487 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:40,487 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:14:40,495 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:40,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51830 deadline: 1731960940493, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:40,495 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:40,495 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:40,495 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:40,495 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:14:40,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51886 deadline: 1731960940493, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:40,495 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:40,495 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:40,496 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:14:40,495 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:40,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51868 deadline: 1731960940493, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:40,496 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:40,496 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:40,496 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:14:40,539 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=95, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/.tmp/A/bac0ca3c4b4f491cbdcab34ab03fec7d 2024-11-18T20:14:40,545 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/.tmp/B/aa29dd98ec0749c8ae547bcc0c1e6aff is 50, key is test_row_0/B:col10/1731960880115/Put/seqid=0 2024-11-18T20:14:40,547 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742408_1584 (size=12001) 2024-11-18T20:14:40,559 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35963 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-18T20:14:40,560 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6. 2024-11-18T20:14:40,560 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6. as already flushing 2024-11-18T20:14:40,560 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6. 2024-11-18T20:14:40,560 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => 3c05f66f0b4334a1f317161bff1a01e6, NAME => 'TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:14:40,560 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => 3c05f66f0b4334a1f317161bff1a01e6, NAME => 'TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:14:40,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41277 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3c05f66f0b4334a1f317161bff1a01e6, NAME => 'TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3c05f66f0b4334a1f317161bff1a01e6, NAME => 'TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:14:40,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-18T20:14:40,712 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35963 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-18T20:14:40,712 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6. 2024-11-18T20:14:40,712 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6. as already flushing 2024-11-18T20:14:40,712 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6. 2024-11-18T20:14:40,712 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => 3c05f66f0b4334a1f317161bff1a01e6, NAME => 'TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:14:40,712 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => 3c05f66f0b4334a1f317161bff1a01e6, NAME => 'TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:14:40,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41277 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3c05f66f0b4334a1f317161bff1a01e6, NAME => 'TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3c05f66f0b4334a1f317161bff1a01e6, NAME => 'TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:14:40,795 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:40,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51816 deadline: 1731960940793, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:40,795 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:40,795 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:40,795 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:40,796 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:14:40,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51828 deadline: 1731960940793, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:40,796 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:40,796 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:40,796 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:14:40,805 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:40,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51830 deadline: 1731960940803, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:40,805 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:40,805 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:40,805 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:14:40,806 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:40,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51886 deadline: 1731960940803, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:40,806 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:40,806 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:40,806 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:40,806 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:14:40,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51868 deadline: 1731960940803, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:40,806 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:40,806 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:40,806 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:14:40,865 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35963 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-18T20:14:40,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6. 2024-11-18T20:14:40,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6. as already flushing 2024-11-18T20:14:40,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6. 2024-11-18T20:14:40,865 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => 3c05f66f0b4334a1f317161bff1a01e6, NAME => 'TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:14:40,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => 3c05f66f0b4334a1f317161bff1a01e6, NAME => 'TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:14:40,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41277 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3c05f66f0b4334a1f317161bff1a01e6, NAME => 'TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3c05f66f0b4334a1f317161bff1a01e6, NAME => 'TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:14:40,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-18T20:14:40,948 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=95 (bloomFilter=true), to=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/.tmp/B/aa29dd98ec0749c8ae547bcc0c1e6aff 2024-11-18T20:14:40,955 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/.tmp/C/12fb2791886f49f5948d8a0049a0a20e is 50, key is test_row_0/C:col10/1731960880115/Put/seqid=0 2024-11-18T20:14:40,962 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742409_1585 (size=12001) 2024-11-18T20:14:40,963 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=95 (bloomFilter=true), to=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/.tmp/C/12fb2791886f49f5948d8a0049a0a20e 2024-11-18T20:14:40,966 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/.tmp/A/bac0ca3c4b4f491cbdcab34ab03fec7d as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/A/bac0ca3c4b4f491cbdcab34ab03fec7d 2024-11-18T20:14:40,969 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/A/bac0ca3c4b4f491cbdcab34ab03fec7d, entries=200, sequenceid=95, filesize=38.6 K 2024-11-18T20:14:40,970 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/.tmp/B/aa29dd98ec0749c8ae547bcc0c1e6aff as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/B/aa29dd98ec0749c8ae547bcc0c1e6aff 2024-11-18T20:14:40,973 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/B/aa29dd98ec0749c8ae547bcc0c1e6aff, entries=150, sequenceid=95, filesize=11.7 K 2024-11-18T20:14:40,974 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/.tmp/C/12fb2791886f49f5948d8a0049a0a20e as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/C/12fb2791886f49f5948d8a0049a0a20e 2024-11-18T20:14:40,977 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/C/12fb2791886f49f5948d8a0049a0a20e, entries=150, sequenceid=95, filesize=11.7 K 2024-11-18T20:14:40,977 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=120.76 KB/123660 for 3c05f66f0b4334a1f317161bff1a01e6 in 861ms, sequenceid=95, compaction requested=true 2024-11-18T20:14:40,977 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3c05f66f0b4334a1f317161bff1a01e6: 2024-11-18T20:14:40,978 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3c05f66f0b4334a1f317161bff1a01e6:A, priority=-2147483648, current under compaction store size is 1 2024-11-18T20:14:40,978 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T20:14:40,978 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:14:40,978 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3c05f66f0b4334a1f317161bff1a01e6:B, priority=-2147483648, current under compaction store size is 2 2024-11-18T20:14:40,978 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:14:40,978 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T20:14:40,978 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3c05f66f0b4334a1f317161bff1a01e6:C, priority=-2147483648, current under compaction store size is 3 2024-11-18T20:14:40,978 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T20:14:40,978 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101562 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T20:14:40,978 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T20:14:40,978 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HStore(1541): 3c05f66f0b4334a1f317161bff1a01e6/A is initiating minor compaction (all files) 2024-11-18T20:14:40,978 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HStore(1541): 3c05f66f0b4334a1f317161bff1a01e6/B is initiating minor compaction (all files) 2024-11-18T20:14:40,978 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3c05f66f0b4334a1f317161bff1a01e6/A in TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6. 2024-11-18T20:14:40,978 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/A/c5cd93c24e68438495fb8ff6b04e83b5, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/A/c0cf7c7af19743ee921a3326887d970e, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/A/bac0ca3c4b4f491cbdcab34ab03fec7d] into tmpdir=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/.tmp, totalSize=99.2 K 2024-11-18T20:14:40,979 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6. 2024-11-18T20:14:40,979 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6. files: [hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/A/c5cd93c24e68438495fb8ff6b04e83b5, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/A/c0cf7c7af19743ee921a3326887d970e, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/A/bac0ca3c4b4f491cbdcab34ab03fec7d] 2024-11-18T20:14:40,979 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] compactions.Compactor(225): Compacting c5cd93c24e68438495fb8ff6b04e83b5, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=55, earliestPutTs=1731960876118 2024-11-18T20:14:40,979 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3c05f66f0b4334a1f317161bff1a01e6/B in TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6. 2024-11-18T20:14:40,979 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/B/8c89c95908234d6d84f6bf1f371176a0, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/B/a9969320227b4298a623918c098a470f, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/B/aa29dd98ec0749c8ae547bcc0c1e6aff] into tmpdir=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/.tmp, totalSize=35.3 K 2024-11-18T20:14:40,980 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] compactions.Compactor(225): Compacting c0cf7c7af19743ee921a3326887d970e, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1731960876772 2024-11-18T20:14:40,980 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.Compactor(225): Compacting 8c89c95908234d6d84f6bf1f371176a0, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=55, earliestPutTs=1731960876118 2024-11-18T20:14:40,980 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] compactions.Compactor(225): Compacting bac0ca3c4b4f491cbdcab34ab03fec7d, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1731960878966 2024-11-18T20:14:40,980 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.Compactor(225): Compacting a9969320227b4298a623918c098a470f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1731960876772 2024-11-18T20:14:40,980 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.Compactor(225): Compacting aa29dd98ec0749c8ae547bcc0c1e6aff, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1731960878973 2024-11-18T20:14:40,985 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3c05f66f0b4334a1f317161bff1a01e6#B#compaction#500 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:14:40,986 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/.tmp/B/406477df55174c6492832fb032e707c6 is 50, key is test_row_0/B:col10/1731960880115/Put/seqid=0 2024-11-18T20:14:40,989 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=3c05f66f0b4334a1f317161bff1a01e6] 2024-11-18T20:14:40,991 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411180ef69a65acb6447eae8e6492e132b393_3c05f66f0b4334a1f317161bff1a01e6 store=[table=TestAcidGuarantees family=A region=3c05f66f0b4334a1f317161bff1a01e6] 2024-11-18T20:14:40,993 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411180ef69a65acb6447eae8e6492e132b393_3c05f66f0b4334a1f317161bff1a01e6, store=[table=TestAcidGuarantees family=A region=3c05f66f0b4334a1f317161bff1a01e6] 2024-11-18T20:14:40,994 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411180ef69a65acb6447eae8e6492e132b393_3c05f66f0b4334a1f317161bff1a01e6 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=3c05f66f0b4334a1f317161bff1a01e6] 2024-11-18T20:14:41,002 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742410_1586 (size=12207) 2024-11-18T20:14:41,004 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742411_1587 (size=4469) 2024-11-18T20:14:41,005 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3c05f66f0b4334a1f317161bff1a01e6#A#compaction#501 average throughput is 1.53 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:14:41,005 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/.tmp/A/68ef0ab4ff4843beaddcba61b314befb is 175, key is test_row_0/A:col10/1731960880115/Put/seqid=0 2024-11-18T20:14:41,009 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742412_1588 (size=31161) 2024-11-18T20:14:41,017 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35963 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-18T20:14:41,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6. 2024-11-18T20:14:41,018 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2902): Flushing 3c05f66f0b4334a1f317161bff1a01e6 3/3 column families, dataSize=120.76 KB heapSize=317.16 KB 2024-11-18T20:14:41,024 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411188a545dd1f1aa48df9a4151cdbc5c128a_3c05f66f0b4334a1f317161bff1a01e6 is 50, key is test_row_0/A:col10/1731960880160/Put/seqid=0 2024-11-18T20:14:41,032 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742413_1589 (size=12154) 2024-11-18T20:14:41,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(8855): Flush requested on 3c05f66f0b4334a1f317161bff1a01e6 2024-11-18T20:14:41,305 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6. as already flushing 2024-11-18T20:14:41,323 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:41,323 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:41,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51868 deadline: 1731960941319, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:41,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51886 deadline: 1731960941318, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:41,324 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:41,324 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:41,324 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:14:41,324 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:41,324 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:41,324 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:14:41,327 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:41,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51828 deadline: 1731960941322, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:41,328 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:41,328 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:41,328 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:41,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51816 deadline: 1731960941323, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:41,328 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:14:41,328 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:41,328 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:41,328 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:14:41,328 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:41,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51830 deadline: 1731960941324, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:41,328 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:41,328 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:41,328 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:14:41,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-18T20:14:41,406 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/.tmp/B/406477df55174c6492832fb032e707c6 as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/B/406477df55174c6492832fb032e707c6 2024-11-18T20:14:41,410 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3c05f66f0b4334a1f317161bff1a01e6/B of 3c05f66f0b4334a1f317161bff1a01e6 into 406477df55174c6492832fb032e707c6(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:14:41,410 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3c05f66f0b4334a1f317161bff1a01e6: 2024-11-18T20:14:41,410 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., storeName=3c05f66f0b4334a1f317161bff1a01e6/B, priority=13, startTime=1731960880978; duration=0sec 2024-11-18T20:14:41,410 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T20:14:41,410 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3c05f66f0b4334a1f317161bff1a01e6:B 2024-11-18T20:14:41,410 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T20:14:41,411 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T20:14:41,411 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HStore(1541): 3c05f66f0b4334a1f317161bff1a01e6/C is initiating minor compaction (all files) 2024-11-18T20:14:41,411 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3c05f66f0b4334a1f317161bff1a01e6/C in TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6. 2024-11-18T20:14:41,411 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/C/fb61bacd061b4933b2c51aec4c23222e, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/C/076e25d583474d2ab57a817a1aeff2d2, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/C/12fb2791886f49f5948d8a0049a0a20e] into tmpdir=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/.tmp, totalSize=35.3 K 2024-11-18T20:14:41,413 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.Compactor(225): Compacting fb61bacd061b4933b2c51aec4c23222e, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=55, earliestPutTs=1731960876118 2024-11-18T20:14:41,413 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.Compactor(225): Compacting 076e25d583474d2ab57a817a1aeff2d2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1731960876772 2024-11-18T20:14:41,413 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.Compactor(225): Compacting 12fb2791886f49f5948d8a0049a0a20e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1731960878973 2024-11-18T20:14:41,414 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/.tmp/A/68ef0ab4ff4843beaddcba61b314befb as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/A/68ef0ab4ff4843beaddcba61b314befb 2024-11-18T20:14:41,418 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3c05f66f0b4334a1f317161bff1a01e6/A of 3c05f66f0b4334a1f317161bff1a01e6 into 68ef0ab4ff4843beaddcba61b314befb(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:14:41,418 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3c05f66f0b4334a1f317161bff1a01e6: 2024-11-18T20:14:41,418 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., storeName=3c05f66f0b4334a1f317161bff1a01e6/A, priority=13, startTime=1731960880977; duration=0sec 2024-11-18T20:14:41,418 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:14:41,418 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3c05f66f0b4334a1f317161bff1a01e6:A 2024-11-18T20:14:41,420 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3c05f66f0b4334a1f317161bff1a01e6#C#compaction#503 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:14:41,420 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/.tmp/C/9075c6f7a13e4565b98ab50c0d4c270a is 50, key is test_row_0/C:col10/1731960880115/Put/seqid=0 2024-11-18T20:14:41,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,434 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:41,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51816 deadline: 1731960941433, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:41,435 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:41,435 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:41,435 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:14:41,435 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:41,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51830 deadline: 1731960941433, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:41,435 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:41,435 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:41,435 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:41,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51868 deadline: 1731960941433, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:41,435 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:14:41,435 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:41,435 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:41,435 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:41,435 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:14:41,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51828 deadline: 1731960941434, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:41,436 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:41,436 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:41,436 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:41,436 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:14:41,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51886 deadline: 1731960941434, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:41,436 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:41,436 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:41,436 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:14:41,437 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411188a545dd1f1aa48df9a4151cdbc5c128a_3c05f66f0b4334a1f317161bff1a01e6 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411188a545dd1f1aa48df9a4151cdbc5c128a_3c05f66f0b4334a1f317161bff1a01e6 2024-11-18T20:14:41,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/.tmp/A/7a4faf779614496bb5280a805576a54f, store: [table=TestAcidGuarantees family=A region=3c05f66f0b4334a1f317161bff1a01e6] 2024-11-18T20:14:41,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/.tmp/A/7a4faf779614496bb5280a805576a54f is 175, key is test_row_0/A:col10/1731960880160/Put/seqid=0 2024-11-18T20:14:41,462 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742414_1590 (size=12207) 2024-11-18T20:14:41,466 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742415_1591 (size=30955) 2024-11-18T20:14:41,467 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=116, memsize=40.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/.tmp/A/7a4faf779614496bb5280a805576a54f 2024-11-18T20:14:41,467 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/.tmp/C/9075c6f7a13e4565b98ab50c0d4c270a as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/C/9075c6f7a13e4565b98ab50c0d4c270a 2024-11-18T20:14:41,474 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3c05f66f0b4334a1f317161bff1a01e6/C of 3c05f66f0b4334a1f317161bff1a01e6 into 9075c6f7a13e4565b98ab50c0d4c270a(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:14:41,474 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3c05f66f0b4334a1f317161bff1a01e6: 2024-11-18T20:14:41,474 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., storeName=3c05f66f0b4334a1f317161bff1a01e6/C, priority=13, startTime=1731960880978; duration=0sec 2024-11-18T20:14:41,474 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:14:41,474 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3c05f66f0b4334a1f317161bff1a01e6:C 2024-11-18T20:14:41,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/.tmp/B/5bd7b758d4b8439d92e68727dd47e52a is 50, key is test_row_0/B:col10/1731960880160/Put/seqid=0 2024-11-18T20:14:41,483 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742416_1592 (size=12001) 2024-11-18T20:14:41,483 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=116 (bloomFilter=true), to=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/.tmp/B/5bd7b758d4b8439d92e68727dd47e52a 2024-11-18T20:14:41,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/.tmp/C/5da7863cd7b842d79f8d1203b850ccfc is 50, key is test_row_0/C:col10/1731960880160/Put/seqid=0 2024-11-18T20:14:41,494 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742417_1593 (size=12001) 2024-11-18T20:14:41,495 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=116 (bloomFilter=true), to=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/.tmp/C/5da7863cd7b842d79f8d1203b850ccfc 2024-11-18T20:14:41,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/.tmp/A/7a4faf779614496bb5280a805576a54f as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/A/7a4faf779614496bb5280a805576a54f 2024-11-18T20:14:41,504 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/A/7a4faf779614496bb5280a805576a54f, entries=150, sequenceid=116, filesize=30.2 K 2024-11-18T20:14:41,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/.tmp/B/5bd7b758d4b8439d92e68727dd47e52a as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/B/5bd7b758d4b8439d92e68727dd47e52a 2024-11-18T20:14:41,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,509 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/B/5bd7b758d4b8439d92e68727dd47e52a, entries=150, sequenceid=116, filesize=11.7 K 2024-11-18T20:14:41,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/.tmp/C/5da7863cd7b842d79f8d1203b850ccfc as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/C/5da7863cd7b842d79f8d1203b850ccfc 2024-11-18T20:14:41,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,514 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/C/5da7863cd7b842d79f8d1203b850ccfc, entries=150, sequenceid=116, filesize=11.7 K 2024-11-18T20:14:41,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,515 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~317.11 KB/324720, currentSize=87.22 KB/89310 for 3c05f66f0b4334a1f317161bff1a01e6 in 496ms, sequenceid=116, compaction requested=false 2024-11-18T20:14:41,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,515 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2603): Flush status journal for 3c05f66f0b4334a1f317161bff1a01e6: 2024-11-18T20:14:41,515 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6. 2024-11-18T20:14:41,515 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=159 2024-11-18T20:14:41,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41277 {}] master.HMaster(4169): Remote procedure done, pid=159 2024-11-18T20:14:41,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,518 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=159, resume processing ppid=158 2024-11-18T20:14:41,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,518 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=159, ppid=158, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2610 sec 2024-11-18T20:14:41,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,519 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=158, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees in 1.2650 sec 2024-11-18T20:14:41,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(8855): Flush requested on 3c05f66f0b4334a1f317161bff1a01e6 2024-11-18T20:14:41,649 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-18T20:14:41,649 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3c05f66f0b4334a1f317161bff1a01e6 3/3 column families, dataSize=93.93 KB heapSize=246.84 KB 2024-11-18T20:14:41,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,657 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111801e55b058ebd4968b197873aca5a74c9_3c05f66f0b4334a1f317161bff1a01e6 is 50, key is test_row_0/A:col10/1731960881648/Put/seqid=0 2024-11-18T20:14:41,662 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742418_1594 (size=14794) 2024-11-18T20:14:41,663 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:41,667 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111801e55b058ebd4968b197873aca5a74c9_3c05f66f0b4334a1f317161bff1a01e6 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111801e55b058ebd4968b197873aca5a74c9_3c05f66f0b4334a1f317161bff1a01e6 2024-11-18T20:14:41,668 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/.tmp/A/fadc8d61b3c1491986ff885366edcad8, store: [table=TestAcidGuarantees family=A region=3c05f66f0b4334a1f317161bff1a01e6] 2024-11-18T20:14:41,668 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/.tmp/A/fadc8d61b3c1491986ff885366edcad8 is 175, key is test_row_0/A:col10/1731960881648/Put/seqid=0 2024-11-18T20:14:41,672 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742419_1595 (size=39749) 2024-11-18T20:14:41,674 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:41,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51830 deadline: 1731960941668, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:41,674 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:41,675 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:41,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51868 deadline: 1731960941668, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:41,675 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:41,675 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:14:41,675 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:41,675 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:41,675 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:41,675 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:14:41,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51886 deadline: 1731960941669, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:41,675 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:41,675 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:41,675 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:14:41,676 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:41,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51816 deadline: 1731960941672, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:41,677 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:41,677 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:41,677 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:14:41,677 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:41,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51828 deadline: 1731960941673, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:41,677 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:41,677 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:41,677 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:14:41,785 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:41,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51830 deadline: 1731960941783, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:41,785 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:41,785 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:41,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51886 deadline: 1731960941783, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:41,785 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:41,785 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:14:41,785 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:41,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51868 deadline: 1731960941783, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:41,785 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:41,785 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:41,785 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:14:41,786 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:41,786 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:41,786 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:41,786 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:14:41,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51816 deadline: 1731960941783, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:41,786 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:41,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51828 deadline: 1731960941783, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:41,786 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:41,786 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:41,786 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:14:41,786 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:41,786 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:41,786 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:14:41,995 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:41,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51830 deadline: 1731960941993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:41,995 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:41,995 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:41,995 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:41,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51886 deadline: 1731960941993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:41,995 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:14:41,995 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:41,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51868 deadline: 1731960941993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:41,996 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:41,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51816 deadline: 1731960941993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:41,996 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:41,996 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:41,996 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:41,996 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:14:41,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51828 deadline: 1731960941993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:41,996 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:41,996 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:41,996 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:14:41,996 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:41,996 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:41,996 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:41,996 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:41,996 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:14:41,996 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:14:42,073 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=137, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/.tmp/A/fadc8d61b3c1491986ff885366edcad8 2024-11-18T20:14:42,078 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/.tmp/B/c119b21db4d44d429f3a5461f614fd49 is 50, key is test_row_0/B:col10/1731960881648/Put/seqid=0 2024-11-18T20:14:42,081 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742420_1596 (size=12151) 2024-11-18T20:14:42,081 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=137 (bloomFilter=true), to=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/.tmp/B/c119b21db4d44d429f3a5461f614fd49 2024-11-18T20:14:42,087 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/.tmp/C/e5004530da284f7ebf23a6e67d708647 is 50, key is test_row_0/C:col10/1731960881648/Put/seqid=0 2024-11-18T20:14:42,091 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742421_1597 (size=12151) 2024-11-18T20:14:42,091 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=137 (bloomFilter=true), to=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/.tmp/C/e5004530da284f7ebf23a6e67d708647 2024-11-18T20:14:42,095 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/.tmp/A/fadc8d61b3c1491986ff885366edcad8 as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/A/fadc8d61b3c1491986ff885366edcad8 2024-11-18T20:14:42,097 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/A/fadc8d61b3c1491986ff885366edcad8, entries=200, sequenceid=137, filesize=38.8 K 2024-11-18T20:14:42,098 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/.tmp/B/c119b21db4d44d429f3a5461f614fd49 as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/B/c119b21db4d44d429f3a5461f614fd49 2024-11-18T20:14:42,101 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/B/c119b21db4d44d429f3a5461f614fd49, entries=150, sequenceid=137, filesize=11.9 K 2024-11-18T20:14:42,102 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/.tmp/C/e5004530da284f7ebf23a6e67d708647 as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/C/e5004530da284f7ebf23a6e67d708647 2024-11-18T20:14:42,105 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/C/e5004530da284f7ebf23a6e67d708647, entries=150, sequenceid=137, filesize=11.9 K 2024-11-18T20:14:42,106 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~264.38 KB/270720, currentSize=100.63 KB/103050 for 3c05f66f0b4334a1f317161bff1a01e6 in 457ms, sequenceid=137, compaction requested=true 2024-11-18T20:14:42,106 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3c05f66f0b4334a1f317161bff1a01e6: 2024-11-18T20:14:42,106 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3c05f66f0b4334a1f317161bff1a01e6:A, priority=-2147483648, current under compaction store size is 1 2024-11-18T20:14:42,106 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:14:42,106 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T20:14:42,106 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3c05f66f0b4334a1f317161bff1a01e6:B, priority=-2147483648, current under compaction store size is 2 2024-11-18T20:14:42,106 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:14:42,106 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T20:14:42,106 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3c05f66f0b4334a1f317161bff1a01e6:C, priority=-2147483648, current under compaction store size is 3 2024-11-18T20:14:42,106 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T20:14:42,107 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101865 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T20:14:42,107 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HStore(1541): 3c05f66f0b4334a1f317161bff1a01e6/A is initiating minor compaction (all files) 2024-11-18T20:14:42,107 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3c05f66f0b4334a1f317161bff1a01e6/A in TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6. 2024-11-18T20:14:42,107 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36359 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T20:14:42,108 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HStore(1541): 3c05f66f0b4334a1f317161bff1a01e6/B is initiating minor compaction (all files) 2024-11-18T20:14:42,108 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/A/68ef0ab4ff4843beaddcba61b314befb, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/A/7a4faf779614496bb5280a805576a54f, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/A/fadc8d61b3c1491986ff885366edcad8] into tmpdir=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/.tmp, totalSize=99.5 K 2024-11-18T20:14:42,108 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3c05f66f0b4334a1f317161bff1a01e6/B in TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6. 2024-11-18T20:14:42,108 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6. 2024-11-18T20:14:42,108 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/B/406477df55174c6492832fb032e707c6, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/B/5bd7b758d4b8439d92e68727dd47e52a, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/B/c119b21db4d44d429f3a5461f614fd49] into tmpdir=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/.tmp, totalSize=35.5 K 2024-11-18T20:14:42,108 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6. files: [hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/A/68ef0ab4ff4843beaddcba61b314befb, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/A/7a4faf779614496bb5280a805576a54f, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/A/fadc8d61b3c1491986ff885366edcad8] 2024-11-18T20:14:42,108 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] compactions.Compactor(225): Compacting 68ef0ab4ff4843beaddcba61b314befb, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1731960878973 2024-11-18T20:14:42,108 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.Compactor(225): Compacting 406477df55174c6492832fb032e707c6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1731960878973 2024-11-18T20:14:42,108 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.Compactor(225): Compacting 5bd7b758d4b8439d92e68727dd47e52a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1731960880160 2024-11-18T20:14:42,108 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7a4faf779614496bb5280a805576a54f, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1731960880160 2024-11-18T20:14:42,109 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.Compactor(225): Compacting c119b21db4d44d429f3a5461f614fd49, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=137, earliestPutTs=1731960881321 2024-11-18T20:14:42,109 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] compactions.Compactor(225): Compacting fadc8d61b3c1491986ff885366edcad8, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=137, earliestPutTs=1731960881321 2024-11-18T20:14:42,114 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=3c05f66f0b4334a1f317161bff1a01e6] 2024-11-18T20:14:42,115 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3c05f66f0b4334a1f317161bff1a01e6#B#compaction#509 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:14:42,116 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/.tmp/B/7bfc490bd6514e9cb83a90b85042b717 is 50, key is test_row_0/B:col10/1731960881648/Put/seqid=0 2024-11-18T20:14:42,116 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241118571afb3fb2804d6b89c0066754b387f1_3c05f66f0b4334a1f317161bff1a01e6 store=[table=TestAcidGuarantees family=A region=3c05f66f0b4334a1f317161bff1a01e6] 2024-11-18T20:14:42,118 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241118571afb3fb2804d6b89c0066754b387f1_3c05f66f0b4334a1f317161bff1a01e6, store=[table=TestAcidGuarantees family=A region=3c05f66f0b4334a1f317161bff1a01e6] 2024-11-18T20:14:42,119 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118571afb3fb2804d6b89c0066754b387f1_3c05f66f0b4334a1f317161bff1a01e6 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=3c05f66f0b4334a1f317161bff1a01e6] 2024-11-18T20:14:42,134 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742422_1598 (size=12459) 2024-11-18T20:14:42,142 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742423_1599 (size=4469) 2024-11-18T20:14:42,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(8855): Flush requested on 3c05f66f0b4334a1f317161bff1a01e6 2024-11-18T20:14:42,305 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-18T20:14:42,305 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3c05f66f0b4334a1f317161bff1a01e6 3/3 column families, dataSize=107.34 KB heapSize=282 KB 2024-11-18T20:14:42,316 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411188b71096f664f4c1f96722ec22d4dd256_3c05f66f0b4334a1f317161bff1a01e6 is 50, key is test_row_0/A:col10/1731960881670/Put/seqid=0 2024-11-18T20:14:42,320 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742424_1600 (size=12304) 2024-11-18T20:14:42,334 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:42,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51886 deadline: 1731960942326, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:42,334 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:42,334 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:42,334 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:42,335 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:14:42,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51816 deadline: 1731960942326, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:42,335 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:42,335 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:42,335 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:42,335 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:14:42,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51868 deadline: 1731960942328, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:42,335 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:42,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51828 deadline: 1731960942328, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:42,335 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:42,335 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:42,335 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:14:42,336 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:42,336 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:42,336 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:14:42,339 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:42,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51830 deadline: 1731960942334, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:42,341 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:42,341 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:42,341 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:14:42,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-18T20:14:42,404 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-18T20:14:42,404 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41277 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-18T20:14:42,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41277 {}] procedure2.ProcedureExecutor(1139): Stored pid=160, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees 2024-11-18T20:14:42,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-18T20:14:42,406 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=160, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-18T20:14:42,407 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=160, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-18T20:14:42,407 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=161, ppid=160, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-18T20:14:42,444 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:42,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51886 deadline: 1731960942443, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:42,444 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:42,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51816 deadline: 1731960942443, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:42,444 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:42,444 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:42,444 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:14:42,444 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:42,444 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:42,444 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:14:42,444 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:42,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51868 deadline: 1731960942443, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:42,445 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:42,445 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:42,445 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:14:42,445 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:42,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51830 deadline: 1731960942443, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:42,445 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:42,445 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:42,445 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:14:42,445 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:42,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51828 deadline: 1731960942444, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:42,445 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:42,445 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:42,445 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:14:42,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-18T20:14:42,538 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/.tmp/B/7bfc490bd6514e9cb83a90b85042b717 as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/B/7bfc490bd6514e9cb83a90b85042b717 2024-11-18T20:14:42,541 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3c05f66f0b4334a1f317161bff1a01e6/B of 3c05f66f0b4334a1f317161bff1a01e6 into 7bfc490bd6514e9cb83a90b85042b717(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:14:42,541 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3c05f66f0b4334a1f317161bff1a01e6: 2024-11-18T20:14:42,541 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., storeName=3c05f66f0b4334a1f317161bff1a01e6/B, priority=13, startTime=1731960882106; duration=0sec 2024-11-18T20:14:42,541 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T20:14:42,541 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3c05f66f0b4334a1f317161bff1a01e6:B 2024-11-18T20:14:42,542 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T20:14:42,542 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36359 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T20:14:42,543 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HStore(1541): 3c05f66f0b4334a1f317161bff1a01e6/C is initiating minor compaction (all files) 2024-11-18T20:14:42,543 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3c05f66f0b4334a1f317161bff1a01e6/C in TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6. 2024-11-18T20:14:42,543 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/C/9075c6f7a13e4565b98ab50c0d4c270a, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/C/5da7863cd7b842d79f8d1203b850ccfc, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/C/e5004530da284f7ebf23a6e67d708647] into tmpdir=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/.tmp, totalSize=35.5 K 2024-11-18T20:14:42,543 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.Compactor(225): Compacting 9075c6f7a13e4565b98ab50c0d4c270a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1731960878973 2024-11-18T20:14:42,543 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.Compactor(225): Compacting 5da7863cd7b842d79f8d1203b850ccfc, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1731960880160 2024-11-18T20:14:42,543 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3c05f66f0b4334a1f317161bff1a01e6#A#compaction#510 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:14:42,543 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.Compactor(225): Compacting e5004530da284f7ebf23a6e67d708647, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=137, earliestPutTs=1731960881321 2024-11-18T20:14:42,544 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/.tmp/A/d4c3c30b89fb41e78fbf4142ea756050 is 175, key is test_row_0/A:col10/1731960881648/Put/seqid=0 2024-11-18T20:14:42,546 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742425_1601 (size=31413) 2024-11-18T20:14:42,549 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3c05f66f0b4334a1f317161bff1a01e6#C#compaction#512 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:14:42,550 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/.tmp/C/a679dcf47f314857a7455151ac18552a is 50, key is test_row_0/C:col10/1731960881648/Put/seqid=0 2024-11-18T20:14:42,553 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742426_1602 (size=12459) 2024-11-18T20:14:42,558 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35963 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-18T20:14:42,558 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6. 2024-11-18T20:14:42,558 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6. as already flushing 2024-11-18T20:14:42,559 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6. 2024-11-18T20:14:42,559 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 3c05f66f0b4334a1f317161bff1a01e6, NAME => 'TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:14:42,559 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 3c05f66f0b4334a1f317161bff1a01e6, NAME => 'TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:14:42,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41277 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3c05f66f0b4334a1f317161bff1a01e6, NAME => 'TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3c05f66f0b4334a1f317161bff1a01e6, NAME => 'TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:14:42,655 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:42,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51886 deadline: 1731960942653, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:42,655 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:42,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51816 deadline: 1731960942653, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:42,655 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:42,655 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:42,655 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:14:42,655 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:42,655 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:42,655 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:42,655 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:14:42,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51868 deadline: 1731960942653, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:42,655 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:42,655 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:42,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51830 deadline: 1731960942653, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:42,655 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:42,656 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:14:42,656 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:42,656 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:42,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51828 deadline: 1731960942653, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:42,656 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:42,656 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:14:42,656 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:42,656 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:42,656 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:14:42,711 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35963 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-18T20:14:42,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6. 2024-11-18T20:14:42,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6. as already flushing 2024-11-18T20:14:42,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6. 2024-11-18T20:14:42,711 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 3c05f66f0b4334a1f317161bff1a01e6, NAME => 'TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:14:42,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 3c05f66f0b4334a1f317161bff1a01e6, NAME => 'TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:14:42,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41277 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3c05f66f0b4334a1f317161bff1a01e6, NAME => 'TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3c05f66f0b4334a1f317161bff1a01e6, NAME => 'TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:14:42,721 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:42,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-18T20:14:42,724 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411188b71096f664f4c1f96722ec22d4dd256_3c05f66f0b4334a1f317161bff1a01e6 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411188b71096f664f4c1f96722ec22d4dd256_3c05f66f0b4334a1f317161bff1a01e6 2024-11-18T20:14:42,724 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/.tmp/A/50275b6b2e1f4e1a858a21e6f6327e0b, store: [table=TestAcidGuarantees family=A region=3c05f66f0b4334a1f317161bff1a01e6] 2024-11-18T20:14:42,725 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/.tmp/A/50275b6b2e1f4e1a858a21e6f6327e0b is 175, key is test_row_0/A:col10/1731960881670/Put/seqid=0 2024-11-18T20:14:42,728 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742427_1603 (size=31105) 2024-11-18T20:14:42,730 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=157, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/.tmp/A/50275b6b2e1f4e1a858a21e6f6327e0b 2024-11-18T20:14:42,738 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/.tmp/B/f5b46135f83d4485901d300da5d36d41 is 50, key is test_row_0/B:col10/1731960881670/Put/seqid=0 2024-11-18T20:14:42,742 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742428_1604 (size=12151) 2024-11-18T20:14:42,742 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=157 (bloomFilter=true), to=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/.tmp/B/f5b46135f83d4485901d300da5d36d41 2024-11-18T20:14:42,748 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/.tmp/C/ec2aa7a8035344f7a9070cc3333480ed is 50, key is test_row_0/C:col10/1731960881670/Put/seqid=0 2024-11-18T20:14:42,751 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742429_1605 (size=12151) 2024-11-18T20:14:42,752 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=157 (bloomFilter=true), to=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/.tmp/C/ec2aa7a8035344f7a9070cc3333480ed 2024-11-18T20:14:42,760 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/.tmp/A/50275b6b2e1f4e1a858a21e6f6327e0b as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/A/50275b6b2e1f4e1a858a21e6f6327e0b 2024-11-18T20:14:42,764 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/A/50275b6b2e1f4e1a858a21e6f6327e0b, entries=150, sequenceid=157, filesize=30.4 K 2024-11-18T20:14:42,764 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/.tmp/B/f5b46135f83d4485901d300da5d36d41 as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/B/f5b46135f83d4485901d300da5d36d41 2024-11-18T20:14:42,767 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/B/f5b46135f83d4485901d300da5d36d41, entries=150, sequenceid=157, filesize=11.9 K 2024-11-18T20:14:42,767 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/.tmp/C/ec2aa7a8035344f7a9070cc3333480ed as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/C/ec2aa7a8035344f7a9070cc3333480ed 2024-11-18T20:14:42,770 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/C/ec2aa7a8035344f7a9070cc3333480ed, entries=150, sequenceid=157, filesize=11.9 K 2024-11-18T20:14:42,771 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~299.53 KB/306720, currentSize=87.22 KB/89310 for 3c05f66f0b4334a1f317161bff1a01e6 in 466ms, sequenceid=157, compaction requested=false 2024-11-18T20:14:42,771 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3c05f66f0b4334a1f317161bff1a01e6: 2024-11-18T20:14:42,863 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35963 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-18T20:14:42,863 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6. 2024-11-18T20:14:42,863 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2902): Flushing 3c05f66f0b4334a1f317161bff1a01e6 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2024-11-18T20:14:42,868 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411187c649b997f2a45968d92ef991ae57b60_3c05f66f0b4334a1f317161bff1a01e6 is 50, key is test_row_0/A:col10/1731960882326/Put/seqid=0 2024-11-18T20:14:42,871 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742430_1606 (size=12304) 2024-11-18T20:14:42,950 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/.tmp/A/d4c3c30b89fb41e78fbf4142ea756050 as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/A/d4c3c30b89fb41e78fbf4142ea756050 2024-11-18T20:14:42,953 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3c05f66f0b4334a1f317161bff1a01e6/A of 3c05f66f0b4334a1f317161bff1a01e6 into d4c3c30b89fb41e78fbf4142ea756050(size=30.7 K), total size for store is 61.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:14:42,953 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3c05f66f0b4334a1f317161bff1a01e6: 2024-11-18T20:14:42,953 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., storeName=3c05f66f0b4334a1f317161bff1a01e6/A, priority=13, startTime=1731960882106; duration=0sec 2024-11-18T20:14:42,953 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:14:42,953 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3c05f66f0b4334a1f317161bff1a01e6:A 2024-11-18T20:14:42,956 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/.tmp/C/a679dcf47f314857a7455151ac18552a as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/C/a679dcf47f314857a7455151ac18552a 2024-11-18T20:14:42,960 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3c05f66f0b4334a1f317161bff1a01e6/C of 3c05f66f0b4334a1f317161bff1a01e6 into a679dcf47f314857a7455151ac18552a(size=12.2 K), total size for store is 24.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:14:42,960 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3c05f66f0b4334a1f317161bff1a01e6: 2024-11-18T20:14:42,960 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., storeName=3c05f66f0b4334a1f317161bff1a01e6/C, priority=13, startTime=1731960882106; duration=0sec 2024-11-18T20:14:42,960 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:14:42,960 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3c05f66f0b4334a1f317161bff1a01e6:C 2024-11-18T20:14:42,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(8855): Flush requested on 3c05f66f0b4334a1f317161bff1a01e6 2024-11-18T20:14:42,965 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6. as already flushing 2024-11-18T20:14:42,982 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:42,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51828 deadline: 1731960942977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:42,983 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:42,983 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:42,983 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:42,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51868 deadline: 1731960942978, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:42,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51830 deadline: 1731960942977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:42,983 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:42,983 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:14:42,983 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:42,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51886 deadline: 1731960942980, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:42,983 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:42,983 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:42,983 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:42,983 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:42,983 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:14:42,983 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:14:42,983 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:42,983 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:42,983 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:14:42,986 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:42,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51816 deadline: 1731960942982, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:42,986 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:42,986 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:42,986 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:14:43,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-18T20:14:43,094 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:43,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51828 deadline: 1731960943093, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:43,095 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:43,094 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:43,095 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:43,095 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:14:43,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51830 deadline: 1731960943093, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:43,095 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:43,095 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:43,095 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:43,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51868 deadline: 1731960943093, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:43,095 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:14:43,095 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:43,095 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:43,095 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:14:43,096 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:43,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51816 deadline: 1731960943094, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:43,097 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:43,097 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:43,097 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:43,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51886 deadline: 1731960943094, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:43,097 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:14:43,097 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:43,097 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:43,097 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:14:43,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:43,274 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411187c649b997f2a45968d92ef991ae57b60_3c05f66f0b4334a1f317161bff1a01e6 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411187c649b997f2a45968d92ef991ae57b60_3c05f66f0b4334a1f317161bff1a01e6 2024-11-18T20:14:43,275 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/.tmp/A/b767b9cf07ee4754b1a3d3de58e739b7, store: [table=TestAcidGuarantees family=A region=3c05f66f0b4334a1f317161bff1a01e6] 2024-11-18T20:14:43,275 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/.tmp/A/b767b9cf07ee4754b1a3d3de58e739b7 is 175, key is test_row_0/A:col10/1731960882326/Put/seqid=0 2024-11-18T20:14:43,278 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742431_1607 (size=31105) 2024-11-18T20:14:43,305 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:43,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51828 deadline: 1731960943303, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:43,305 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:43,305 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:43,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51830 deadline: 1731960943303, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:43,305 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:43,305 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:14:43,305 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:43,306 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:43,306 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:14:43,306 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:43,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51868 deadline: 1731960943303, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:43,306 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:43,306 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:43,306 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:43,306 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:14:43,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51886 deadline: 1731960943304, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:43,307 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:43,307 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:43,307 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:43,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51816 deadline: 1731960943304, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:43,307 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:14:43,307 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:43,307 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:43,307 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:14:43,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-18T20:14:43,615 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:43,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51828 deadline: 1731960943613, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:43,615 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:43,615 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:43,615 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:14:43,616 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:43,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51830 deadline: 1731960943613, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:43,616 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:43,616 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:43,616 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:14:43,616 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:43,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51886 deadline: 1731960943613, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:43,616 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:43,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51868 deadline: 1731960943613, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:43,616 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:43,616 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:43,616 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:43,616 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:14:43,616 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:43,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51816 deadline: 1731960943613, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:43,617 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:43,617 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:14:43,617 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:43,617 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:43,617 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:14:43,679 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=174, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/.tmp/A/b767b9cf07ee4754b1a3d3de58e739b7 2024-11-18T20:14:43,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/.tmp/B/81fe69dbf1c5439fb9f37f6996d1d746 is 50, key is test_row_0/B:col10/1731960882326/Put/seqid=0 2024-11-18T20:14:43,692 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742432_1608 (size=12151) 2024-11-18T20:14:44,092 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=174 (bloomFilter=true), to=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/.tmp/B/81fe69dbf1c5439fb9f37f6996d1d746 2024-11-18T20:14:44,102 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/.tmp/C/146c39e37c1a460088e2ab1f2cb0f202 is 50, key is test_row_0/C:col10/1731960882326/Put/seqid=0 2024-11-18T20:14:44,105 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742433_1609 (size=12151) 2024-11-18T20:14:44,105 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=174 (bloomFilter=true), to=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/.tmp/C/146c39e37c1a460088e2ab1f2cb0f202 2024-11-18T20:14:44,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/.tmp/A/b767b9cf07ee4754b1a3d3de58e739b7 as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/A/b767b9cf07ee4754b1a3d3de58e739b7 2024-11-18T20:14:44,110 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/A/b767b9cf07ee4754b1a3d3de58e739b7, entries=150, sequenceid=174, filesize=30.4 K 2024-11-18T20:14:44,111 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/.tmp/B/81fe69dbf1c5439fb9f37f6996d1d746 as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/B/81fe69dbf1c5439fb9f37f6996d1d746 2024-11-18T20:14:44,113 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/B/81fe69dbf1c5439fb9f37f6996d1d746, entries=150, sequenceid=174, filesize=11.9 K 2024-11-18T20:14:44,114 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/.tmp/C/146c39e37c1a460088e2ab1f2cb0f202 as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/C/146c39e37c1a460088e2ab1f2cb0f202 2024-11-18T20:14:44,116 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/C/146c39e37c1a460088e2ab1f2cb0f202, entries=150, sequenceid=174, filesize=11.9 K 2024-11-18T20:14:44,117 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=114.05 KB/116790 for 3c05f66f0b4334a1f317161bff1a01e6 in 1254ms, sequenceid=174, compaction requested=true 2024-11-18T20:14:44,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2603): Flush status journal for 3c05f66f0b4334a1f317161bff1a01e6: 2024-11-18T20:14:44,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6. 2024-11-18T20:14:44,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=161 2024-11-18T20:14:44,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41277 {}] master.HMaster(4169): Remote procedure done, pid=161 2024-11-18T20:14:44,120 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=161, resume processing ppid=160 2024-11-18T20:14:44,120 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=161, ppid=160, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7110 sec 2024-11-18T20:14:44,121 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=160, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees in 1.7160 sec 2024-11-18T20:14:44,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(8855): Flush requested on 3c05f66f0b4334a1f317161bff1a01e6 2024-11-18T20:14:44,127 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-18T20:14:44,127 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3c05f66f0b4334a1f317161bff1a01e6 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-18T20:14:44,133 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411182aeeea248d4c4403a6b917056ac2830b_3c05f66f0b4334a1f317161bff1a01e6 is 50, key is test_row_0/A:col10/1731960882979/Put/seqid=0 2024-11-18T20:14:44,140 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:44,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51868 deadline: 1731960944136, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:44,141 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:44,141 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:44,141 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:14:44,144 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:44,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51886 deadline: 1731960944139, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:44,145 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:44,145 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:44,145 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:14:44,145 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:44,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51830 deadline: 1731960944139, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:44,145 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:44,145 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:44,145 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:44,145 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:14:44,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51816 deadline: 1731960944139, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:44,145 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:44,145 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:44,145 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:14:44,145 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:44,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51828 deadline: 1731960944140, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:44,145 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:44,145 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:44,146 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:14:44,147 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742434_1610 (size=12304) 2024-11-18T20:14:44,245 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:44,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51868 deadline: 1731960944243, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:44,245 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:44,245 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:44,245 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:14:44,255 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:44,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51886 deadline: 1731960944253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:44,255 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:44,255 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:44,255 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:44,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51830 deadline: 1731960944253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:44,255 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:14:44,255 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:44,255 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:44,255 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:44,255 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:44,255 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:14:44,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51816 deadline: 1731960944253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:44,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51828 deadline: 1731960944253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:44,256 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:44,256 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:44,256 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:44,256 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:44,256 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:14:44,256 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:14:44,453 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:44,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51868 deadline: 1731960944452, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:44,454 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:44,454 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:44,454 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:14:44,464 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:44,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51830 deadline: 1731960944463, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:44,464 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:44,464 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:44,464 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:44,464 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:14:44,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51886 deadline: 1731960944463, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:44,464 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:44,464 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:44,464 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:44,464 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:14:44,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51828 deadline: 1731960944463, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:44,465 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:44,465 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:44,465 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:14:44,465 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:44,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51816 deadline: 1731960944463, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:44,465 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:44,465 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:44,465 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:14:44,548 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:44,550 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411182aeeea248d4c4403a6b917056ac2830b_3c05f66f0b4334a1f317161bff1a01e6 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411182aeeea248d4c4403a6b917056ac2830b_3c05f66f0b4334a1f317161bff1a01e6 2024-11-18T20:14:44,551 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/.tmp/A/95369b3c7711447ea70140c840e1da0f, store: [table=TestAcidGuarantees family=A region=3c05f66f0b4334a1f317161bff1a01e6] 2024-11-18T20:14:44,552 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/.tmp/A/95369b3c7711447ea70140c840e1da0f is 175, key is test_row_0/A:col10/1731960882979/Put/seqid=0 2024-11-18T20:14:44,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-18T20:14:44,553 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-18T20:14:44,554 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41277 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-18T20:14:44,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41277 {}] procedure2.ProcedureExecutor(1139): Stored pid=162, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees 2024-11-18T20:14:44,555 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742435_1611 (size=31105) 2024-11-18T20:14:44,555 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=162, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-18T20:14:44,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-18T20:14:44,556 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=162, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-18T20:14:44,556 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=163, ppid=162, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-18T20:14:44,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-18T20:14:44,707 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35963 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-18T20:14:44,708 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6. 2024-11-18T20:14:44,708 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6. as already flushing 2024-11-18T20:14:44,708 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6. 2024-11-18T20:14:44,708 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => 3c05f66f0b4334a1f317161bff1a01e6, NAME => 'TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:14:44,708 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => 3c05f66f0b4334a1f317161bff1a01e6, NAME => 'TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:14:44,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41277 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3c05f66f0b4334a1f317161bff1a01e6, NAME => 'TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3c05f66f0b4334a1f317161bff1a01e6, NAME => 'TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:14:44,763 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:44,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51868 deadline: 1731960944762, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:44,764 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:44,764 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:44,764 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:14:44,775 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:44,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51830 deadline: 1731960944773, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:44,775 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:44,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51886 deadline: 1731960944773, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:44,775 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:44,775 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:44,775 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:14:44,775 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:44,775 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:44,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51828 deadline: 1731960944773, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:44,775 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:44,775 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:14:44,775 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:44,775 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:44,775 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:44,775 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:14:44,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51816 deadline: 1731960944773, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:44,776 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:44,776 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:44,776 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:14:44,860 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35963 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-18T20:14:44,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6. 2024-11-18T20:14:44,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6. as already flushing 2024-11-18T20:14:44,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6. 2024-11-18T20:14:44,860 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => 3c05f66f0b4334a1f317161bff1a01e6, NAME => 'TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:14:44,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => 3c05f66f0b4334a1f317161bff1a01e6, NAME => 'TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:14:44,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41277 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3c05f66f0b4334a1f317161bff1a01e6, NAME => 'TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3c05f66f0b4334a1f317161bff1a01e6, NAME => 'TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:14:44,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-18T20:14:44,955 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=199, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/.tmp/A/95369b3c7711447ea70140c840e1da0f 2024-11-18T20:14:44,960 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/.tmp/B/fbf5a743d11e41cea554aa2b3deccc57 is 50, key is test_row_0/B:col10/1731960882979/Put/seqid=0 2024-11-18T20:14:44,962 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742436_1612 (size=12151) 2024-11-18T20:14:45,012 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35963 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-18T20:14:45,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6. 2024-11-18T20:14:45,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6. as already flushing 2024-11-18T20:14:45,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6. 2024-11-18T20:14:45,013 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => 3c05f66f0b4334a1f317161bff1a01e6, NAME => 'TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:14:45,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => 3c05f66f0b4334a1f317161bff1a01e6, NAME => 'TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:14:45,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41277 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3c05f66f0b4334a1f317161bff1a01e6, NAME => 'TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3c05f66f0b4334a1f317161bff1a01e6, NAME => 'TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:14:45,165 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35963 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-18T20:14:45,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6. 2024-11-18T20:14:45,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6. as already flushing 2024-11-18T20:14:45,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6. 2024-11-18T20:14:45,165 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => 3c05f66f0b4334a1f317161bff1a01e6, NAME => 'TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:14:45,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => 3c05f66f0b4334a1f317161bff1a01e6, NAME => 'TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:14:45,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41277 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3c05f66f0b4334a1f317161bff1a01e6, NAME => 'TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3c05f66f0b4334a1f317161bff1a01e6, NAME => 'TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:14:45,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-18T20:14:45,276 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:45,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51868 deadline: 1731960945273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:45,277 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:45,277 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:45,277 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:14:45,286 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:45,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51830 deadline: 1731960945283, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:45,286 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:45,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51886 deadline: 1731960945283, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:45,287 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:45,287 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:45,287 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:45,287 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:14:45,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51816 deadline: 1731960945283, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:45,287 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:45,287 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:45,287 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:45,287 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:14:45,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51828 deadline: 1731960945283, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:45,287 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:45,287 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:45,287 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:14:45,287 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:45,287 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:45,287 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:14:45,317 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35963 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-18T20:14:45,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6. 2024-11-18T20:14:45,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6. as already flushing 2024-11-18T20:14:45,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6. 2024-11-18T20:14:45,317 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => 3c05f66f0b4334a1f317161bff1a01e6, NAME => 'TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:14:45,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => 3c05f66f0b4334a1f317161bff1a01e6, NAME => 'TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:14:45,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41277 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3c05f66f0b4334a1f317161bff1a01e6, NAME => 'TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3c05f66f0b4334a1f317161bff1a01e6, NAME => 'TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:14:45,362 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=199 (bloomFilter=true), to=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/.tmp/B/fbf5a743d11e41cea554aa2b3deccc57 2024-11-18T20:14:45,368 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/.tmp/C/debde896eb0447b7a6f386d72bdc43e4 is 50, key is test_row_0/C:col10/1731960882979/Put/seqid=0 2024-11-18T20:14:45,371 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742437_1613 (size=12151) 2024-11-18T20:14:45,470 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35963 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-18T20:14:45,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6. 2024-11-18T20:14:45,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6. as already flushing 2024-11-18T20:14:45,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6. 2024-11-18T20:14:45,470 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => 3c05f66f0b4334a1f317161bff1a01e6, NAME => 'TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:14:45,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => 3c05f66f0b4334a1f317161bff1a01e6, NAME => 'TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:14:45,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41277 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3c05f66f0b4334a1f317161bff1a01e6, NAME => 'TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3c05f66f0b4334a1f317161bff1a01e6, NAME => 'TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:14:45,622 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35963 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-18T20:14:45,622 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6. 2024-11-18T20:14:45,622 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6. as already flushing 2024-11-18T20:14:45,622 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6. 2024-11-18T20:14:45,623 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => 3c05f66f0b4334a1f317161bff1a01e6, NAME => 'TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:14:45,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => 3c05f66f0b4334a1f317161bff1a01e6, NAME => 'TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:14:45,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41277 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3c05f66f0b4334a1f317161bff1a01e6, NAME => 'TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3c05f66f0b4334a1f317161bff1a01e6, NAME => 'TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:14:45,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-18T20:14:45,772 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=199 (bloomFilter=true), to=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/.tmp/C/debde896eb0447b7a6f386d72bdc43e4 2024-11-18T20:14:45,775 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35963 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-18T20:14:45,775 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6. 2024-11-18T20:14:45,775 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6. as already flushing 2024-11-18T20:14:45,775 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6. 2024-11-18T20:14:45,775 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => 3c05f66f0b4334a1f317161bff1a01e6, NAME => 'TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:14:45,775 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/.tmp/A/95369b3c7711447ea70140c840e1da0f as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/A/95369b3c7711447ea70140c840e1da0f 2024-11-18T20:14:45,775 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => 3c05f66f0b4334a1f317161bff1a01e6, NAME => 'TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:14:45,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41277 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3c05f66f0b4334a1f317161bff1a01e6, NAME => 'TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3c05f66f0b4334a1f317161bff1a01e6, NAME => 'TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:14:45,778 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/A/95369b3c7711447ea70140c840e1da0f, entries=150, sequenceid=199, filesize=30.4 K 2024-11-18T20:14:45,778 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/.tmp/B/fbf5a743d11e41cea554aa2b3deccc57 as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/B/fbf5a743d11e41cea554aa2b3deccc57 2024-11-18T20:14:45,781 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/B/fbf5a743d11e41cea554aa2b3deccc57, entries=150, sequenceid=199, filesize=11.9 K 2024-11-18T20:14:45,781 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/.tmp/C/debde896eb0447b7a6f386d72bdc43e4 as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/C/debde896eb0447b7a6f386d72bdc43e4 2024-11-18T20:14:45,789 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/C/debde896eb0447b7a6f386d72bdc43e4, entries=150, sequenceid=199, filesize=11.9 K 2024-11-18T20:14:45,789 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=73.80 KB/75570 for 3c05f66f0b4334a1f317161bff1a01e6 in 1662ms, sequenceid=199, compaction requested=true 2024-11-18T20:14:45,789 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3c05f66f0b4334a1f317161bff1a01e6: 2024-11-18T20:14:45,789 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3c05f66f0b4334a1f317161bff1a01e6:A, priority=-2147483648, current under compaction store size is 1 2024-11-18T20:14:45,790 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:14:45,790 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-18T20:14:45,790 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3c05f66f0b4334a1f317161bff1a01e6:B, priority=-2147483648, current under compaction store size is 2 2024-11-18T20:14:45,790 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:14:45,790 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-18T20:14:45,790 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3c05f66f0b4334a1f317161bff1a01e6:C, priority=-2147483648, current under compaction store size is 3 2024-11-18T20:14:45,790 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T20:14:45,790 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48912 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-18T20:14:45,790 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 124728 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-18T20:14:45,790 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HStore(1541): 3c05f66f0b4334a1f317161bff1a01e6/A is initiating minor compaction (all files) 2024-11-18T20:14:45,790 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HStore(1541): 3c05f66f0b4334a1f317161bff1a01e6/B is initiating minor compaction (all files) 2024-11-18T20:14:45,790 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3c05f66f0b4334a1f317161bff1a01e6/A in TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6. 2024-11-18T20:14:45,790 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3c05f66f0b4334a1f317161bff1a01e6/B in TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6. 2024-11-18T20:14:45,791 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/A/d4c3c30b89fb41e78fbf4142ea756050, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/A/50275b6b2e1f4e1a858a21e6f6327e0b, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/A/b767b9cf07ee4754b1a3d3de58e739b7, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/A/95369b3c7711447ea70140c840e1da0f] into tmpdir=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/.tmp, totalSize=121.8 K 2024-11-18T20:14:45,791 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/B/7bfc490bd6514e9cb83a90b85042b717, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/B/f5b46135f83d4485901d300da5d36d41, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/B/81fe69dbf1c5439fb9f37f6996d1d746, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/B/fbf5a743d11e41cea554aa2b3deccc57] into tmpdir=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/.tmp, totalSize=47.8 K 2024-11-18T20:14:45,791 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6. 2024-11-18T20:14:45,791 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6. files: [hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/A/d4c3c30b89fb41e78fbf4142ea756050, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/A/50275b6b2e1f4e1a858a21e6f6327e0b, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/A/b767b9cf07ee4754b1a3d3de58e739b7, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/A/95369b3c7711447ea70140c840e1da0f] 2024-11-18T20:14:45,791 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.Compactor(225): Compacting 7bfc490bd6514e9cb83a90b85042b717, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=137, earliestPutTs=1731960881321 2024-11-18T20:14:45,791 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] compactions.Compactor(225): Compacting d4c3c30b89fb41e78fbf4142ea756050, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=137, earliestPutTs=1731960881321 2024-11-18T20:14:45,791 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.Compactor(225): Compacting f5b46135f83d4485901d300da5d36d41, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1731960881670 2024-11-18T20:14:45,791 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] compactions.Compactor(225): Compacting 50275b6b2e1f4e1a858a21e6f6327e0b, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1731960881670 2024-11-18T20:14:45,791 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.Compactor(225): Compacting 81fe69dbf1c5439fb9f37f6996d1d746, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=174, earliestPutTs=1731960882326 2024-11-18T20:14:45,792 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.Compactor(225): Compacting fbf5a743d11e41cea554aa2b3deccc57, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=199, earliestPutTs=1731960882979 2024-11-18T20:14:45,792 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] compactions.Compactor(225): Compacting b767b9cf07ee4754b1a3d3de58e739b7, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=174, earliestPutTs=1731960882326 2024-11-18T20:14:45,792 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] compactions.Compactor(225): Compacting 95369b3c7711447ea70140c840e1da0f, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=199, earliestPutTs=1731960882979 2024-11-18T20:14:45,797 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=3c05f66f0b4334a1f317161bff1a01e6] 2024-11-18T20:14:45,798 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3c05f66f0b4334a1f317161bff1a01e6#B#compaction#521 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:14:45,799 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/.tmp/B/ecd351c7261743d3a8daaa3ceef0efb6 is 50, key is test_row_0/B:col10/1731960882979/Put/seqid=0 2024-11-18T20:14:45,807 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241118543fca99ee6e4245b9c74b3c8df113f6_3c05f66f0b4334a1f317161bff1a01e6 store=[table=TestAcidGuarantees family=A region=3c05f66f0b4334a1f317161bff1a01e6] 2024-11-18T20:14:45,807 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742438_1614 (size=12595) 2024-11-18T20:14:45,809 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241118543fca99ee6e4245b9c74b3c8df113f6_3c05f66f0b4334a1f317161bff1a01e6, store=[table=TestAcidGuarantees family=A region=3c05f66f0b4334a1f317161bff1a01e6] 2024-11-18T20:14:45,809 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118543fca99ee6e4245b9c74b3c8df113f6_3c05f66f0b4334a1f317161bff1a01e6 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=3c05f66f0b4334a1f317161bff1a01e6] 2024-11-18T20:14:45,812 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742439_1615 (size=4469) 2024-11-18T20:14:45,813 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/.tmp/B/ecd351c7261743d3a8daaa3ceef0efb6 as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/B/ecd351c7261743d3a8daaa3ceef0efb6 2024-11-18T20:14:45,816 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3c05f66f0b4334a1f317161bff1a01e6#A#compaction#522 average throughput is 1.29 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:14:45,817 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 3c05f66f0b4334a1f317161bff1a01e6/B of 3c05f66f0b4334a1f317161bff1a01e6 into ecd351c7261743d3a8daaa3ceef0efb6(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:14:45,817 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3c05f66f0b4334a1f317161bff1a01e6: 2024-11-18T20:14:45,817 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., storeName=3c05f66f0b4334a1f317161bff1a01e6/B, priority=12, startTime=1731960885790; duration=0sec 2024-11-18T20:14:45,817 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T20:14:45,817 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3c05f66f0b4334a1f317161bff1a01e6:B 2024-11-18T20:14:45,817 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-18T20:14:45,817 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/.tmp/A/63cf1986db11465fa711efde1833e8ae is 175, key is test_row_0/A:col10/1731960882979/Put/seqid=0 2024-11-18T20:14:45,817 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48912 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-18T20:14:45,818 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HStore(1541): 3c05f66f0b4334a1f317161bff1a01e6/C is initiating minor compaction (all files) 2024-11-18T20:14:45,818 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3c05f66f0b4334a1f317161bff1a01e6/C in TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6. 2024-11-18T20:14:45,818 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/C/a679dcf47f314857a7455151ac18552a, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/C/ec2aa7a8035344f7a9070cc3333480ed, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/C/146c39e37c1a460088e2ab1f2cb0f202, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/C/debde896eb0447b7a6f386d72bdc43e4] into tmpdir=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/.tmp, totalSize=47.8 K 2024-11-18T20:14:45,818 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.Compactor(225): Compacting a679dcf47f314857a7455151ac18552a, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=137, earliestPutTs=1731960881321 2024-11-18T20:14:45,818 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.Compactor(225): Compacting ec2aa7a8035344f7a9070cc3333480ed, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1731960881670 2024-11-18T20:14:45,819 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.Compactor(225): Compacting 146c39e37c1a460088e2ab1f2cb0f202, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=174, earliestPutTs=1731960882326 2024-11-18T20:14:45,819 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.Compactor(225): Compacting debde896eb0447b7a6f386d72bdc43e4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=199, earliestPutTs=1731960882979 2024-11-18T20:14:45,829 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3c05f66f0b4334a1f317161bff1a01e6#C#compaction#523 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:14:45,829 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/.tmp/C/e091ead8117f4549933b792712b3f574 is 50, key is test_row_0/C:col10/1731960882979/Put/seqid=0 2024-11-18T20:14:45,830 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742440_1616 (size=31549) 2024-11-18T20:14:45,832 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742441_1617 (size=12595) 2024-11-18T20:14:45,835 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/.tmp/C/e091ead8117f4549933b792712b3f574 as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/C/e091ead8117f4549933b792712b3f574 2024-11-18T20:14:45,839 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 3c05f66f0b4334a1f317161bff1a01e6/C of 3c05f66f0b4334a1f317161bff1a01e6 into e091ead8117f4549933b792712b3f574(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:14:45,839 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3c05f66f0b4334a1f317161bff1a01e6: 2024-11-18T20:14:45,839 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., storeName=3c05f66f0b4334a1f317161bff1a01e6/C, priority=12, startTime=1731960885790; duration=0sec 2024-11-18T20:14:45,840 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:14:45,840 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3c05f66f0b4334a1f317161bff1a01e6:C 2024-11-18T20:14:45,927 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35963 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-18T20:14:45,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6. 2024-11-18T20:14:45,927 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2902): Flushing 3c05f66f0b4334a1f317161bff1a01e6 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-18T20:14:45,932 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118cb4c0c1af09b48f1853504265975fc96_3c05f66f0b4334a1f317161bff1a01e6 is 50, key is test_row_0/A:col10/1731960884139/Put/seqid=0 2024-11-18T20:14:45,935 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742442_1618 (size=12304) 2024-11-18T20:14:46,231 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/.tmp/A/63cf1986db11465fa711efde1833e8ae as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/A/63cf1986db11465fa711efde1833e8ae 2024-11-18T20:14:46,235 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 3c05f66f0b4334a1f317161bff1a01e6/A of 3c05f66f0b4334a1f317161bff1a01e6 into 63cf1986db11465fa711efde1833e8ae(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:14:46,235 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3c05f66f0b4334a1f317161bff1a01e6: 2024-11-18T20:14:46,235 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., storeName=3c05f66f0b4334a1f317161bff1a01e6/A, priority=12, startTime=1731960885789; duration=0sec 2024-11-18T20:14:46,235 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:14:46,235 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3c05f66f0b4334a1f317161bff1a01e6:A 2024-11-18T20:14:46,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(8855): Flush requested on 3c05f66f0b4334a1f317161bff1a01e6 2024-11-18T20:14:46,294 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6. as already flushing 2024-11-18T20:14:46,315 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:46,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51816 deadline: 1731960946311, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:46,315 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:46,315 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:46,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51828 deadline: 1731960946311, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:46,315 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:46,315 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:14:46,315 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:46,315 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:46,315 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:14:46,315 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:46,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51868 deadline: 1731960946312, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:46,316 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:46,316 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:46,316 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:46,316 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:14:46,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51830 deadline: 1731960946313, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:46,316 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:46,316 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:46,316 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:14:46,318 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:46,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51886 deadline: 1731960946315, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:46,318 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:46,318 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:46,318 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:14:46,336 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:46,338 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118cb4c0c1af09b48f1853504265975fc96_3c05f66f0b4334a1f317161bff1a01e6 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118cb4c0c1af09b48f1853504265975fc96_3c05f66f0b4334a1f317161bff1a01e6 2024-11-18T20:14:46,339 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/.tmp/A/ccd0ea0e74be44e5b46bac4695497976, store: [table=TestAcidGuarantees family=A region=3c05f66f0b4334a1f317161bff1a01e6] 2024-11-18T20:14:46,340 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/.tmp/A/ccd0ea0e74be44e5b46bac4695497976 is 175, key is test_row_0/A:col10/1731960884139/Put/seqid=0 2024-11-18T20:14:46,342 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742443_1619 (size=31105) 2024-11-18T20:14:46,345 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=215, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/.tmp/A/ccd0ea0e74be44e5b46bac4695497976 2024-11-18T20:14:46,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/.tmp/B/67c934fed476488bada5d8c0951fef8f is 50, key is test_row_0/B:col10/1731960884139/Put/seqid=0 2024-11-18T20:14:46,361 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742444_1620 (size=12151) 2024-11-18T20:14:46,424 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:46,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51816 deadline: 1731960946423, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:46,424 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:46,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51828 deadline: 1731960946423, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:46,424 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:46,424 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:46,424 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:14:46,424 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:46,424 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:46,424 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:14:46,425 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:46,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51830 deadline: 1731960946423, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:46,426 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:46,426 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:46,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51868 deadline: 1731960946423, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:46,426 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:46,426 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:14:46,426 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:46,426 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:46,426 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:14:46,426 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:46,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51886 deadline: 1731960946424, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:46,426 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:46,426 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:46,426 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:14:46,635 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:46,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51816 deadline: 1731960946633, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:46,635 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:46,635 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:46,635 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:46,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51828 deadline: 1731960946633, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:46,635 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:14:46,636 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:46,636 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:46,636 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:14:46,636 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:46,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51830 deadline: 1731960946633, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:46,637 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:46,637 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:46,637 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:46,637 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:14:46,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51868 deadline: 1731960946633, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:46,637 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:46,637 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:46,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51886 deadline: 1731960946634, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:46,637 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:46,637 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:14:46,637 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:46,637 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:46,637 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:14:46,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-18T20:14:46,761 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=215 (bloomFilter=true), to=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/.tmp/B/67c934fed476488bada5d8c0951fef8f 2024-11-18T20:14:46,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/.tmp/C/8df4e12f03b34b4ba51e40550ef94729 is 50, key is test_row_0/C:col10/1731960884139/Put/seqid=0 2024-11-18T20:14:46,769 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742445_1621 (size=12151) 2024-11-18T20:14:46,944 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:46,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51816 deadline: 1731960946943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:46,945 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:46,945 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:46,945 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:46,945 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:14:46,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51830 deadline: 1731960946943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:46,945 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:46,945 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:46,945 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:46,945 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:14:46,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51828 deadline: 1731960946943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:46,945 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:46,945 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:46,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51868 deadline: 1731960946943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:46,945 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:46,946 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:14:46,946 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:46,946 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:46,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51886 deadline: 1731960946943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:46,946 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:46,946 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:14:46,946 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:46,946 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:46,946 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:14:47,169 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=215 (bloomFilter=true), to=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/.tmp/C/8df4e12f03b34b4ba51e40550ef94729 2024-11-18T20:14:47,173 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/.tmp/A/ccd0ea0e74be44e5b46bac4695497976 as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/A/ccd0ea0e74be44e5b46bac4695497976 2024-11-18T20:14:47,175 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/A/ccd0ea0e74be44e5b46bac4695497976, entries=150, sequenceid=215, filesize=30.4 K 2024-11-18T20:14:47,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/.tmp/B/67c934fed476488bada5d8c0951fef8f as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/B/67c934fed476488bada5d8c0951fef8f 2024-11-18T20:14:47,178 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/B/67c934fed476488bada5d8c0951fef8f, entries=150, sequenceid=215, filesize=11.9 K 2024-11-18T20:14:47,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/.tmp/C/8df4e12f03b34b4ba51e40550ef94729 as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/C/8df4e12f03b34b4ba51e40550ef94729 2024-11-18T20:14:47,181 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/C/8df4e12f03b34b4ba51e40550ef94729, entries=150, sequenceid=215, filesize=11.9 K 2024-11-18T20:14:47,182 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for 3c05f66f0b4334a1f317161bff1a01e6 in 1255ms, sequenceid=215, compaction requested=false 2024-11-18T20:14:47,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2603): Flush status journal for 3c05f66f0b4334a1f317161bff1a01e6: 2024-11-18T20:14:47,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6. 2024-11-18T20:14:47,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=163 2024-11-18T20:14:47,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41277 {}] master.HMaster(4169): Remote procedure done, pid=163 2024-11-18T20:14:47,184 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=163, resume processing ppid=162 2024-11-18T20:14:47,184 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=163, ppid=162, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.6270 sec 2024-11-18T20:14:47,186 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=162, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees in 2.6310 sec 2024-11-18T20:14:47,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(8855): Flush requested on 3c05f66f0b4334a1f317161bff1a01e6 2024-11-18T20:14:47,455 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-18T20:14:47,455 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3c05f66f0b4334a1f317161bff1a01e6 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-18T20:14:47,461 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111862004b55c5ad445e94e2e92c6a2988a1_3c05f66f0b4334a1f317161bff1a01e6 is 50, key is test_row_0/A:col10/1731960887454/Put/seqid=0 2024-11-18T20:14:47,464 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742446_1622 (size=12304) 2024-11-18T20:14:47,495 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:47,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51830 deadline: 1731960947490, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:47,495 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:47,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51816 deadline: 1731960947490, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:47,495 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:47,495 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:47,495 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:14:47,495 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:47,495 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:47,495 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:47,495 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:14:47,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51868 deadline: 1731960947491, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:47,496 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:47,496 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:47,496 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:47,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51886 deadline: 1731960947491, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:47,496 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:14:47,496 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:47,496 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:47,496 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:14:47,498 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:47,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51828 deadline: 1731960947495, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:47,498 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:47,498 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:47,498 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:14:47,605 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:47,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51830 deadline: 1731960947603, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:47,605 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:47,605 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:47,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51816 deadline: 1731960947603, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:47,605 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:47,605 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:14:47,605 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:47,605 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:47,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51868 deadline: 1731960947603, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:47,605 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:47,605 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:14:47,605 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:47,605 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:47,605 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:47,605 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:14:47,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51886 deadline: 1731960947603, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:47,606 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:47,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51828 deadline: 1731960947603, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:47,606 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:47,606 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:47,606 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:47,606 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:14:47,606 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:47,606 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:14:47,814 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:47,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51830 deadline: 1731960947813, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:47,814 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:47,814 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:47,814 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:47,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51816 deadline: 1731960947813, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:47,814 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:14:47,815 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:47,815 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:47,815 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:47,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51868 deadline: 1731960947813, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:47,815 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:14:47,815 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:47,815 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:47,815 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:47,815 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:14:47,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51886 deadline: 1731960947813, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:47,815 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:47,815 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:47,815 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:47,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51828 deadline: 1731960947813, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:47,815 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:14:47,815 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:47,815 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:47,815 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:14:47,865 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:47,867 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111862004b55c5ad445e94e2e92c6a2988a1_3c05f66f0b4334a1f317161bff1a01e6 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111862004b55c5ad445e94e2e92c6a2988a1_3c05f66f0b4334a1f317161bff1a01e6 2024-11-18T20:14:47,868 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/.tmp/A/0a9708af986d4f079e03741337c2467a, store: [table=TestAcidGuarantees family=A region=3c05f66f0b4334a1f317161bff1a01e6] 2024-11-18T20:14:47,869 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/.tmp/A/0a9708af986d4f079e03741337c2467a is 175, key is test_row_0/A:col10/1731960887454/Put/seqid=0 2024-11-18T20:14:47,872 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742447_1623 (size=31105) 2024-11-18T20:14:47,872 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=240, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/.tmp/A/0a9708af986d4f079e03741337c2467a 2024-11-18T20:14:47,883 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/.tmp/B/ec9ee31cf4914217bb761a835455025a is 50, key is test_row_0/B:col10/1731960887454/Put/seqid=0 2024-11-18T20:14:47,902 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742448_1624 (size=12151) 2024-11-18T20:14:48,125 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:48,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51830 deadline: 1731960948122, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:48,125 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:48,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51816 deadline: 1731960948122, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:48,125 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:48,125 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:48,125 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:48,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51868 deadline: 1731960948123, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:48,125 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:14:48,125 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:48,125 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:48,125 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:14:48,126 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:48,126 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:48,126 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:48,126 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:14:48,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51886 deadline: 1731960948123, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:48,126 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:48,126 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:48,126 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:14:48,126 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:48,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51828 deadline: 1731960948123, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:48,126 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:48,126 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:48,126 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:14:48,303 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=240 (bloomFilter=true), to=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/.tmp/B/ec9ee31cf4914217bb761a835455025a 2024-11-18T20:14:48,308 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/.tmp/C/b621c19670b34fe6bf1956bf60f127e2 is 50, key is test_row_0/C:col10/1731960887454/Put/seqid=0 2024-11-18T20:14:48,312 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742449_1625 (size=12151) 2024-11-18T20:14:48,634 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:48,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51830 deadline: 1731960948633, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:48,634 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:48,634 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:48,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51816 deadline: 1731960948633, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:48,634 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:48,634 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:14:48,634 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:48,634 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:48,634 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:14:48,635 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:48,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51868 deadline: 1731960948634, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:48,635 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:48,635 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:48,635 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:14:48,635 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:48,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51886 deadline: 1731960948634, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:48,635 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:48,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51828 deadline: 1731960948634, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:48,636 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:48,636 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:48,636 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:14:48,636 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:48,636 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:48,636 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:14:48,712 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=240 (bloomFilter=true), to=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/.tmp/C/b621c19670b34fe6bf1956bf60f127e2 2024-11-18T20:14:48,716 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/.tmp/A/0a9708af986d4f079e03741337c2467a as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/A/0a9708af986d4f079e03741337c2467a 2024-11-18T20:14:48,719 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/A/0a9708af986d4f079e03741337c2467a, entries=150, sequenceid=240, filesize=30.4 K 2024-11-18T20:14:48,720 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/.tmp/B/ec9ee31cf4914217bb761a835455025a as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/B/ec9ee31cf4914217bb761a835455025a 2024-11-18T20:14:48,722 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/B/ec9ee31cf4914217bb761a835455025a, entries=150, sequenceid=240, filesize=11.9 K 2024-11-18T20:14:48,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-18T20:14:48,723 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-18T20:14:48,723 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/.tmp/C/b621c19670b34fe6bf1956bf60f127e2 as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/C/b621c19670b34fe6bf1956bf60f127e2 2024-11-18T20:14:48,723 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41277 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-18T20:14:48,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41277 {}] procedure2.ProcedureExecutor(1139): Stored pid=164, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees 2024-11-18T20:14:48,726 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=164, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-18T20:14:48,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-18T20:14:48,727 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=164, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-18T20:14:48,727 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=165, ppid=164, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-18T20:14:48,731 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/C/b621c19670b34fe6bf1956bf60f127e2, entries=150, sequenceid=240, filesize=11.9 K 2024-11-18T20:14:48,732 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for 3c05f66f0b4334a1f317161bff1a01e6 in 1277ms, sequenceid=240, compaction requested=true 2024-11-18T20:14:48,732 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3c05f66f0b4334a1f317161bff1a01e6: 2024-11-18T20:14:48,732 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3c05f66f0b4334a1f317161bff1a01e6:A, priority=-2147483648, current under compaction store size is 1 2024-11-18T20:14:48,732 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:14:48,732 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T20:14:48,732 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3c05f66f0b4334a1f317161bff1a01e6:B, priority=-2147483648, current under compaction store size is 2 2024-11-18T20:14:48,732 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:14:48,732 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T20:14:48,732 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3c05f66f0b4334a1f317161bff1a01e6:C, priority=-2147483648, current under compaction store size is 3 2024-11-18T20:14:48,732 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T20:14:48,733 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T20:14:48,733 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HStore(1541): 3c05f66f0b4334a1f317161bff1a01e6/B is initiating minor compaction (all files) 2024-11-18T20:14:48,734 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3c05f66f0b4334a1f317161bff1a01e6/B in TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6. 2024-11-18T20:14:48,734 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/B/ecd351c7261743d3a8daaa3ceef0efb6, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/B/67c934fed476488bada5d8c0951fef8f, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/B/ec9ee31cf4914217bb761a835455025a] into tmpdir=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/.tmp, totalSize=36.0 K 2024-11-18T20:14:48,734 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 93759 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T20:14:48,734 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HStore(1541): 3c05f66f0b4334a1f317161bff1a01e6/A is initiating minor compaction (all files) 2024-11-18T20:14:48,734 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3c05f66f0b4334a1f317161bff1a01e6/A in TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6. 2024-11-18T20:14:48,734 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/A/63cf1986db11465fa711efde1833e8ae, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/A/ccd0ea0e74be44e5b46bac4695497976, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/A/0a9708af986d4f079e03741337c2467a] into tmpdir=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/.tmp, totalSize=91.6 K 2024-11-18T20:14:48,734 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6. 2024-11-18T20:14:48,734 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6. files: [hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/A/63cf1986db11465fa711efde1833e8ae, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/A/ccd0ea0e74be44e5b46bac4695497976, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/A/0a9708af986d4f079e03741337c2467a] 2024-11-18T20:14:48,734 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.Compactor(225): Compacting ecd351c7261743d3a8daaa3ceef0efb6, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=199, earliestPutTs=1731960882979 2024-11-18T20:14:48,734 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] compactions.Compactor(225): Compacting 63cf1986db11465fa711efde1833e8ae, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=199, earliestPutTs=1731960882979 2024-11-18T20:14:48,735 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.Compactor(225): Compacting 67c934fed476488bada5d8c0951fef8f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=215, earliestPutTs=1731960884133 2024-11-18T20:14:48,735 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] compactions.Compactor(225): Compacting ccd0ea0e74be44e5b46bac4695497976, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=215, earliestPutTs=1731960884133 2024-11-18T20:14:48,735 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.Compactor(225): Compacting ec9ee31cf4914217bb761a835455025a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=240, earliestPutTs=1731960886310 2024-11-18T20:14:48,735 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0a9708af986d4f079e03741337c2467a, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=240, earliestPutTs=1731960886310 2024-11-18T20:14:48,740 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=3c05f66f0b4334a1f317161bff1a01e6] 2024-11-18T20:14:48,742 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3c05f66f0b4334a1f317161bff1a01e6#B#compaction#531 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:14:48,743 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/.tmp/B/7a492d87e2cf4580b77f9b279fa6ac00 is 50, key is test_row_0/B:col10/1731960887454/Put/seqid=0 2024-11-18T20:14:48,745 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411189c2d6becc75c4ba9bbca6ca5692f7acc_3c05f66f0b4334a1f317161bff1a01e6 store=[table=TestAcidGuarantees family=A region=3c05f66f0b4334a1f317161bff1a01e6] 2024-11-18T20:14:48,747 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411189c2d6becc75c4ba9bbca6ca5692f7acc_3c05f66f0b4334a1f317161bff1a01e6, store=[table=TestAcidGuarantees family=A region=3c05f66f0b4334a1f317161bff1a01e6] 2024-11-18T20:14:48,748 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411189c2d6becc75c4ba9bbca6ca5692f7acc_3c05f66f0b4334a1f317161bff1a01e6 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=3c05f66f0b4334a1f317161bff1a01e6] 2024-11-18T20:14:48,767 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742451_1627 (size=4469) 2024-11-18T20:14:48,774 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742450_1626 (size=12697) 2024-11-18T20:14:48,778 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/.tmp/B/7a492d87e2cf4580b77f9b279fa6ac00 as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/B/7a492d87e2cf4580b77f9b279fa6ac00 2024-11-18T20:14:48,785 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3c05f66f0b4334a1f317161bff1a01e6/B of 3c05f66f0b4334a1f317161bff1a01e6 into 7a492d87e2cf4580b77f9b279fa6ac00(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:14:48,785 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3c05f66f0b4334a1f317161bff1a01e6: 2024-11-18T20:14:48,785 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., storeName=3c05f66f0b4334a1f317161bff1a01e6/B, priority=13, startTime=1731960888732; duration=0sec 2024-11-18T20:14:48,785 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T20:14:48,785 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3c05f66f0b4334a1f317161bff1a01e6:B 2024-11-18T20:14:48,785 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T20:14:48,786 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T20:14:48,786 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HStore(1541): 3c05f66f0b4334a1f317161bff1a01e6/C is initiating minor compaction (all files) 2024-11-18T20:14:48,786 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3c05f66f0b4334a1f317161bff1a01e6/C in TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6. 2024-11-18T20:14:48,786 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/C/e091ead8117f4549933b792712b3f574, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/C/8df4e12f03b34b4ba51e40550ef94729, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/C/b621c19670b34fe6bf1956bf60f127e2] into tmpdir=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/.tmp, totalSize=36.0 K 2024-11-18T20:14:48,786 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.Compactor(225): Compacting e091ead8117f4549933b792712b3f574, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=199, earliestPutTs=1731960882979 2024-11-18T20:14:48,787 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.Compactor(225): Compacting 8df4e12f03b34b4ba51e40550ef94729, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=215, earliestPutTs=1731960884133 2024-11-18T20:14:48,787 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.Compactor(225): Compacting b621c19670b34fe6bf1956bf60f127e2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=240, earliestPutTs=1731960886310 2024-11-18T20:14:48,791 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3c05f66f0b4334a1f317161bff1a01e6#C#compaction#532 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:14:48,792 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/.tmp/C/1d011fd751f24e418bcf2b62a11a0577 is 50, key is test_row_0/C:col10/1731960887454/Put/seqid=0 2024-11-18T20:14:48,794 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742452_1628 (size=12697) 2024-11-18T20:14:48,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-18T20:14:48,880 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35963 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-18T20:14:48,880 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6. 2024-11-18T20:14:48,880 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2902): Flushing 3c05f66f0b4334a1f317161bff1a01e6 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-18T20:14:48,887 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111890c35c326e3a48eab86c0b0076ca184e_3c05f66f0b4334a1f317161bff1a01e6 is 50, key is test_row_0/A:col10/1731960887490/Put/seqid=0 2024-11-18T20:14:48,902 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742453_1629 (size=12304) 2024-11-18T20:14:49,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-18T20:14:49,168 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3c05f66f0b4334a1f317161bff1a01e6#A#compaction#530 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:14:49,168 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/.tmp/A/93d5c4fbaa2a490fb98bca2c9671e41e is 175, key is test_row_0/A:col10/1731960887454/Put/seqid=0 2024-11-18T20:14:49,171 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742454_1630 (size=31651) 2024-11-18T20:14:49,197 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/.tmp/C/1d011fd751f24e418bcf2b62a11a0577 as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/C/1d011fd751f24e418bcf2b62a11a0577 2024-11-18T20:14:49,201 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3c05f66f0b4334a1f317161bff1a01e6/C of 3c05f66f0b4334a1f317161bff1a01e6 into 1d011fd751f24e418bcf2b62a11a0577(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:14:49,201 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3c05f66f0b4334a1f317161bff1a01e6: 2024-11-18T20:14:49,201 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., storeName=3c05f66f0b4334a1f317161bff1a01e6/C, priority=13, startTime=1731960888732; duration=0sec 2024-11-18T20:14:49,201 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:14:49,201 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3c05f66f0b4334a1f317161bff1a01e6:C 2024-11-18T20:14:49,303 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:49,306 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111890c35c326e3a48eab86c0b0076ca184e_3c05f66f0b4334a1f317161bff1a01e6 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111890c35c326e3a48eab86c0b0076ca184e_3c05f66f0b4334a1f317161bff1a01e6 2024-11-18T20:14:49,306 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/.tmp/A/626d51fb026c4c31995c3ef99d01a752, store: [table=TestAcidGuarantees family=A region=3c05f66f0b4334a1f317161bff1a01e6] 2024-11-18T20:14:49,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/.tmp/A/626d51fb026c4c31995c3ef99d01a752 is 175, key is test_row_0/A:col10/1731960887490/Put/seqid=0 2024-11-18T20:14:49,310 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742455_1631 (size=31105) 2024-11-18T20:14:49,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-18T20:14:49,581 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/.tmp/A/93d5c4fbaa2a490fb98bca2c9671e41e as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/A/93d5c4fbaa2a490fb98bca2c9671e41e 2024-11-18T20:14:49,585 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3c05f66f0b4334a1f317161bff1a01e6/A of 3c05f66f0b4334a1f317161bff1a01e6 into 93d5c4fbaa2a490fb98bca2c9671e41e(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:14:49,585 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3c05f66f0b4334a1f317161bff1a01e6: 2024-11-18T20:14:49,585 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., storeName=3c05f66f0b4334a1f317161bff1a01e6/A, priority=13, startTime=1731960888732; duration=0sec 2024-11-18T20:14:49,585 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:14:49,585 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3c05f66f0b4334a1f317161bff1a01e6:A 2024-11-18T20:14:49,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(8855): Flush requested on 3c05f66f0b4334a1f317161bff1a01e6 2024-11-18T20:14:49,645 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6. as already flushing 2024-11-18T20:14:49,691 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:49,691 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:49,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51868 deadline: 1731960949661, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:49,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51828 deadline: 1731960949660, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:49,691 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:49,691 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:49,692 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:14:49,692 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:49,692 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:49,692 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:14:49,694 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:49,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51816 deadline: 1731960949691, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:49,694 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:49,694 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:49,694 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:49,694 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:14:49,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51830 deadline: 1731960949691, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:49,695 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:49,694 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:49,695 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:49,695 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:14:49,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51886 deadline: 1731960949691, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:49,695 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:49,695 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:49,695 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:14:49,710 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=253, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/.tmp/A/626d51fb026c4c31995c3ef99d01a752 2024-11-18T20:14:49,716 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/.tmp/B/613d0a3f582041d5a2c03665cf59e25f is 50, key is test_row_0/B:col10/1731960887490/Put/seqid=0 2024-11-18T20:14:49,718 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742456_1632 (size=12151) 2024-11-18T20:14:49,719 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=253 (bloomFilter=true), to=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/.tmp/B/613d0a3f582041d5a2c03665cf59e25f 2024-11-18T20:14:49,724 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/.tmp/C/d64d9e3da2854481be6e22007a19930c is 50, key is test_row_0/C:col10/1731960887490/Put/seqid=0 2024-11-18T20:14:49,730 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742457_1633 (size=12151) 2024-11-18T20:14:49,794 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:49,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51828 deadline: 1731960949793, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:49,795 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:49,795 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:49,795 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:14:49,804 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:49,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51868 deadline: 1731960949803, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:49,805 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:49,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51816 deadline: 1731960949804, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:49,805 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:49,805 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:49,805 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:14:49,805 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:49,805 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:49,805 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:49,805 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:14:49,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51886 deadline: 1731960949804, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:49,805 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:49,805 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:49,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51830 deadline: 1731960949804, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:49,805 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:49,805 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:14:49,806 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:49,806 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:49,806 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:14:49,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-18T20:14:50,006 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:50,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51828 deadline: 1731960950003, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:50,007 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:50,007 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:50,007 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:14:50,014 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:50,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51868 deadline: 1731960950013, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:50,015 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:50,015 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:50,015 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:14:50,016 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:50,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51886 deadline: 1731960950013, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:50,016 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:50,016 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:50,016 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:14:50,016 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:50,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51816 deadline: 1731960950013, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:50,016 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:50,016 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:50,017 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:14:50,016 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:50,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51830 deadline: 1731960950013, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:50,017 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:50,017 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:50,017 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:14:50,131 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=253 (bloomFilter=true), to=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/.tmp/C/d64d9e3da2854481be6e22007a19930c 2024-11-18T20:14:50,135 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/.tmp/A/626d51fb026c4c31995c3ef99d01a752 as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/A/626d51fb026c4c31995c3ef99d01a752 2024-11-18T20:14:50,138 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/A/626d51fb026c4c31995c3ef99d01a752, entries=150, sequenceid=253, filesize=30.4 K 2024-11-18T20:14:50,139 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/.tmp/B/613d0a3f582041d5a2c03665cf59e25f as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/B/613d0a3f582041d5a2c03665cf59e25f 2024-11-18T20:14:50,144 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/B/613d0a3f582041d5a2c03665cf59e25f, entries=150, sequenceid=253, filesize=11.9 K 2024-11-18T20:14:50,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/.tmp/C/d64d9e3da2854481be6e22007a19930c as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/C/d64d9e3da2854481be6e22007a19930c 2024-11-18T20:14:50,149 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/C/d64d9e3da2854481be6e22007a19930c, entries=150, sequenceid=253, filesize=11.9 K 2024-11-18T20:14:50,149 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=147.60 KB/151140 for 3c05f66f0b4334a1f317161bff1a01e6 in 1269ms, sequenceid=253, compaction requested=false 2024-11-18T20:14:50,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2603): Flush status journal for 3c05f66f0b4334a1f317161bff1a01e6: 2024-11-18T20:14:50,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6. 2024-11-18T20:14:50,150 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=165 2024-11-18T20:14:50,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41277 {}] master.HMaster(4169): Remote procedure done, pid=165 2024-11-18T20:14:50,152 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=165, resume processing ppid=164 2024-11-18T20:14:50,152 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=165, ppid=164, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4240 sec 2024-11-18T20:14:50,153 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=164, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees in 1.4290 sec 2024-11-18T20:14:50,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(8855): Flush requested on 3c05f66f0b4334a1f317161bff1a01e6 2024-11-18T20:14:50,316 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-18T20:14:50,316 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3c05f66f0b4334a1f317161bff1a01e6 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-18T20:14:50,322 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411181e743e72b3604e3dbbaae4b4cdf72be6_3c05f66f0b4334a1f317161bff1a01e6 is 50, key is test_row_0/A:col10/1731960890314/Put/seqid=0 2024-11-18T20:14:50,326 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742458_1634 (size=12454) 2024-11-18T20:14:50,327 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:50,331 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411181e743e72b3604e3dbbaae4b4cdf72be6_3c05f66f0b4334a1f317161bff1a01e6 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411181e743e72b3604e3dbbaae4b4cdf72be6_3c05f66f0b4334a1f317161bff1a01e6 2024-11-18T20:14:50,332 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/.tmp/A/dfe351f4f0734f979741c88c2c6862bf, store: [table=TestAcidGuarantees family=A region=3c05f66f0b4334a1f317161bff1a01e6] 2024-11-18T20:14:50,332 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:50,332 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/.tmp/A/dfe351f4f0734f979741c88c2c6862bf is 175, key is test_row_0/A:col10/1731960890314/Put/seqid=0 2024-11-18T20:14:50,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51868 deadline: 1731960950327, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:50,333 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:50,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51816 deadline: 1731960950328, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:50,333 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:50,333 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:50,333 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:14:50,333 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:50,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51830 deadline: 1731960950328, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:50,333 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:50,333 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:50,333 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:14:50,333 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:50,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51886 deadline: 1731960950330, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:50,334 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:50,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51828 deadline: 1731960950331, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:50,334 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:50,334 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:50,334 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:14:50,334 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:50,334 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:50,334 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:14:50,334 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:50,334 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:50,335 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:14:50,336 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742459_1635 (size=31255) 2024-11-18T20:14:50,444 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:50,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51868 deadline: 1731960950443, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:50,444 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:50,444 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:50,444 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:14:50,445 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:50,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51828 deadline: 1731960950444, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:50,446 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:50,446 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:50,446 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:50,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51816 deadline: 1731960950444, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:50,446 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:14:50,446 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:50,446 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:50,446 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:50,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51830 deadline: 1731960950444, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:50,446 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:14:50,446 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:50,446 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:50,447 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:14:50,447 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:50,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51886 deadline: 1731960950444, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:50,447 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:50,447 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:50,447 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:14:50,655 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:50,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51828 deadline: 1731960950653, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:50,655 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:50,655 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:50,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51816 deadline: 1731960950653, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:50,655 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:50,655 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:14:50,656 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:50,656 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:50,656 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:50,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51868 deadline: 1731960950654, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:50,656 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:14:50,656 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:50,656 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:50,656 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:50,656 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:14:50,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51886 deadline: 1731960950654, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:50,656 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:50,656 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:50,656 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:50,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51830 deadline: 1731960950654, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:50,656 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:14:50,656 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:50,656 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:50,656 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:14:50,737 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=281, memsize=51.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/.tmp/A/dfe351f4f0734f979741c88c2c6862bf 2024-11-18T20:14:50,742 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/.tmp/B/a9a57ebbaf5d48afb588a9e22255bb99 is 50, key is test_row_0/B:col10/1731960890314/Put/seqid=0 2024-11-18T20:14:50,745 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742460_1636 (size=12301) 2024-11-18T20:14:50,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-18T20:14:50,873 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-18T20:14:50,874 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41277 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-18T20:14:50,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41277 {}] procedure2.ProcedureExecutor(1139): Stored pid=166, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=166, table=TestAcidGuarantees 2024-11-18T20:14:50,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-18T20:14:50,875 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=166, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=166, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-18T20:14:50,876 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=166, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=166, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-18T20:14:50,876 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=167, ppid=166, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-18T20:14:50,966 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:50,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51828 deadline: 1731960950963, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:50,970 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:50,970 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:50,970 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:50,970 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:14:50,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51816 deadline: 1731960950963, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:50,971 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:50,971 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:50,971 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:50,971 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:14:50,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51868 deadline: 1731960950963, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:50,971 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:50,971 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:50,971 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:50,971 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:50,971 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:14:50,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51830 deadline: 1731960950963, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:50,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51886 deadline: 1731960950963, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:50,971 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:50,971 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:50,971 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:50,972 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:50,972 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:14:50,972 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:14:50,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-18T20:14:51,028 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35963 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-11-18T20:14:51,028 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6. 2024-11-18T20:14:51,028 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6. as already flushing 2024-11-18T20:14:51,028 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6. 2024-11-18T20:14:51,028 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] handler.RSProcedureHandler(58): pid=167 java.io.IOException: Unable to complete flush {ENCODED => 3c05f66f0b4334a1f317161bff1a01e6, NAME => 'TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:14:51,028 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=167 java.io.IOException: Unable to complete flush {ENCODED => 3c05f66f0b4334a1f317161bff1a01e6, NAME => 'TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:14:51,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41277 {}] master.HMaster(4177): Remote procedure failed, pid=167 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3c05f66f0b4334a1f317161bff1a01e6, NAME => 'TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3c05f66f0b4334a1f317161bff1a01e6, NAME => 'TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:14:51,146 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=281 (bloomFilter=true), to=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/.tmp/B/a9a57ebbaf5d48afb588a9e22255bb99 2024-11-18T20:14:51,151 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/.tmp/C/ee73ca5a49cd4061aa515d3d7ff8dfb9 is 50, key is test_row_0/C:col10/1731960890314/Put/seqid=0 2024-11-18T20:14:51,159 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742461_1637 (size=12301) 2024-11-18T20:14:51,181 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35963 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-11-18T20:14:51,181 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6. 2024-11-18T20:14:51,181 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6. as already flushing 2024-11-18T20:14:51,181 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6. 2024-11-18T20:14:51,181 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] handler.RSProcedureHandler(58): pid=167 java.io.IOException: Unable to complete flush {ENCODED => 3c05f66f0b4334a1f317161bff1a01e6, NAME => 'TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:14:51,181 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=167 java.io.IOException: Unable to complete flush {ENCODED => 3c05f66f0b4334a1f317161bff1a01e6, NAME => 'TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:14:51,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41277 {}] master.HMaster(4177): Remote procedure failed, pid=167 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3c05f66f0b4334a1f317161bff1a01e6, NAME => 'TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3c05f66f0b4334a1f317161bff1a01e6, NAME => 'TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:14:51,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-18T20:14:51,333 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35963 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-11-18T20:14:51,333 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6. 2024-11-18T20:14:51,333 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6. as already flushing 2024-11-18T20:14:51,333 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6. 2024-11-18T20:14:51,334 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] handler.RSProcedureHandler(58): pid=167 java.io.IOException: Unable to complete flush {ENCODED => 3c05f66f0b4334a1f317161bff1a01e6, NAME => 'TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:14:51,334 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=167 java.io.IOException: Unable to complete flush {ENCODED => 3c05f66f0b4334a1f317161bff1a01e6, NAME => 'TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:14:51,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41277 {}] master.HMaster(4177): Remote procedure failed, pid=167 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3c05f66f0b4334a1f317161bff1a01e6, NAME => 'TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3c05f66f0b4334a1f317161bff1a01e6, NAME => 'TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:14:51,483 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:51,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51828 deadline: 1731960951483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:51,484 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:51,484 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:51,484 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:51,484 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:14:51,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51816 deadline: 1731960951483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:51,484 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:51,484 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:51,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51868 deadline: 1731960951483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:51,484 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:51,484 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:14:51,484 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:51,484 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:51,484 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:51,484 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:14:51,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51886 deadline: 1731960951483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:51,485 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:51,485 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:51,485 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:51,485 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:14:51,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51830 deadline: 1731960951483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:51,485 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:51,485 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:51,485 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:14:51,486 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35963 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-11-18T20:14:51,486 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6. 2024-11-18T20:14:51,486 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6. as already flushing 2024-11-18T20:14:51,486 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6. 2024-11-18T20:14:51,486 ERROR [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] handler.RSProcedureHandler(58): pid=167 java.io.IOException: Unable to complete flush {ENCODED => 3c05f66f0b4334a1f317161bff1a01e6, NAME => 'TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:14:51,486 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=167 java.io.IOException: Unable to complete flush {ENCODED => 3c05f66f0b4334a1f317161bff1a01e6, NAME => 'TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:14:51,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41277 {}] master.HMaster(4177): Remote procedure failed, pid=167 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3c05f66f0b4334a1f317161bff1a01e6, NAME => 'TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3c05f66f0b4334a1f317161bff1a01e6, NAME => 'TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T20:14:51,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-18T20:14:51,560 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=281 (bloomFilter=true), to=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/.tmp/C/ee73ca5a49cd4061aa515d3d7ff8dfb9 2024-11-18T20:14:51,563 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/.tmp/A/dfe351f4f0734f979741c88c2c6862bf as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/A/dfe351f4f0734f979741c88c2c6862bf 2024-11-18T20:14:51,565 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/A/dfe351f4f0734f979741c88c2c6862bf, entries=150, sequenceid=281, filesize=30.5 K 2024-11-18T20:14:51,566 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/.tmp/B/a9a57ebbaf5d48afb588a9e22255bb99 as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/B/a9a57ebbaf5d48afb588a9e22255bb99 2024-11-18T20:14:51,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,569 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/B/a9a57ebbaf5d48afb588a9e22255bb99, entries=150, sequenceid=281, filesize=12.0 K 2024-11-18T20:14:51,570 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/.tmp/C/ee73ca5a49cd4061aa515d3d7ff8dfb9 as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/C/ee73ca5a49cd4061aa515d3d7ff8dfb9 2024-11-18T20:14:51,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,573 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/C/ee73ca5a49cd4061aa515d3d7ff8dfb9, entries=150, sequenceid=281, filesize=12.0 K 2024-11-18T20:14:51,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,573 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for 3c05f66f0b4334a1f317161bff1a01e6 in 1257ms, sequenceid=281, compaction requested=true 2024-11-18T20:14:51,574 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3c05f66f0b4334a1f317161bff1a01e6: 2024-11-18T20:14:51,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,574 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T20:14:51,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,574 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3c05f66f0b4334a1f317161bff1a01e6:A, priority=-2147483648, current under compaction store size is 1 2024-11-18T20:14:51,574 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:14:51,574 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T20:14:51,574 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3c05f66f0b4334a1f317161bff1a01e6:B, priority=-2147483648, current under compaction store size is 2 2024-11-18T20:14:51,574 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:14:51,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,574 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3c05f66f0b4334a1f317161bff1a01e6:C, priority=-2147483648, current under compaction store size is 3 2024-11-18T20:14:51,574 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T20:14:51,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,577 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 94011 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T20:14:51,577 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37149 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T20:14:51,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,577 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HStore(1541): 3c05f66f0b4334a1f317161bff1a01e6/B is initiating minor compaction (all files) 2024-11-18T20:14:51,577 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HStore(1541): 3c05f66f0b4334a1f317161bff1a01e6/A is initiating minor compaction (all files) 2024-11-18T20:14:51,578 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3c05f66f0b4334a1f317161bff1a01e6/A in TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6. 2024-11-18T20:14:51,578 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3c05f66f0b4334a1f317161bff1a01e6/B in TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6. 2024-11-18T20:14:51,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,578 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/A/93d5c4fbaa2a490fb98bca2c9671e41e, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/A/626d51fb026c4c31995c3ef99d01a752, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/A/dfe351f4f0734f979741c88c2c6862bf] into tmpdir=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/.tmp, totalSize=91.8 K 2024-11-18T20:14:51,578 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/B/7a492d87e2cf4580b77f9b279fa6ac00, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/B/613d0a3f582041d5a2c03665cf59e25f, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/B/a9a57ebbaf5d48afb588a9e22255bb99] into tmpdir=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/.tmp, totalSize=36.3 K 2024-11-18T20:14:51,578 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6. 2024-11-18T20:14:51,578 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6. files: [hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/A/93d5c4fbaa2a490fb98bca2c9671e41e, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/A/626d51fb026c4c31995c3ef99d01a752, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/A/dfe351f4f0734f979741c88c2c6862bf] 2024-11-18T20:14:51,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,578 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.Compactor(225): Compacting 7a492d87e2cf4580b77f9b279fa6ac00, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=240, earliestPutTs=1731960886310 2024-11-18T20:14:51,578 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] compactions.Compactor(225): Compacting 93d5c4fbaa2a490fb98bca2c9671e41e, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=240, earliestPutTs=1731960886310 2024-11-18T20:14:51,578 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.Compactor(225): Compacting 613d0a3f582041d5a2c03665cf59e25f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=253, earliestPutTs=1731960887461 2024-11-18T20:14:51,578 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] compactions.Compactor(225): Compacting 626d51fb026c4c31995c3ef99d01a752, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=253, earliestPutTs=1731960887461 2024-11-18T20:14:51,578 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] compactions.Compactor(225): Compacting dfe351f4f0734f979741c88c2c6862bf, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=281, earliestPutTs=1731960889659 2024-11-18T20:14:51,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,578 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.Compactor(225): Compacting a9a57ebbaf5d48afb588a9e22255bb99, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=281, earliestPutTs=1731960889659 2024-11-18T20:14:51,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,583 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=3c05f66f0b4334a1f317161bff1a01e6] 2024-11-18T20:14:51,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,584 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3c05f66f0b4334a1f317161bff1a01e6#B#compaction#539 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:14:51,584 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/.tmp/B/f779191614384c52bc2fb5173a40f67d is 50, key is test_row_0/B:col10/1731960890314/Put/seqid=0 2024-11-18T20:14:51,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,585 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024111859c4c4bd13334b97b51e7225dfb3bf41_3c05f66f0b4334a1f317161bff1a01e6 store=[table=TestAcidGuarantees family=A region=3c05f66f0b4334a1f317161bff1a01e6] 2024-11-18T20:14:51,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,587 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024111859c4c4bd13334b97b51e7225dfb3bf41_3c05f66f0b4334a1f317161bff1a01e6, store=[table=TestAcidGuarantees family=A region=3c05f66f0b4334a1f317161bff1a01e6] 2024-11-18T20:14:51,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,587 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111859c4c4bd13334b97b51e7225dfb3bf41_3c05f66f0b4334a1f317161bff1a01e6 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=3c05f66f0b4334a1f317161bff1a01e6] 2024-11-18T20:14:51,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,610 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742463_1639 (size=12949) 2024-11-18T20:14:51,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,616 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742462_1638 (size=4469) 2024-11-18T20:14:51,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,617 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3c05f66f0b4334a1f317161bff1a01e6#A#compaction#540 average throughput is 0.72 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:14:51,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,617 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/.tmp/A/5c879241d007403dbb24d92524c20bb8 is 175, key is test_row_0/A:col10/1731960890314/Put/seqid=0 2024-11-18T20:14:51,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,622 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742464_1640 (size=31903) 2024-11-18T20:14:51,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,626 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/.tmp/A/5c879241d007403dbb24d92524c20bb8 as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/A/5c879241d007403dbb24d92524c20bb8 2024-11-18T20:14:51,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,630 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3c05f66f0b4334a1f317161bff1a01e6/A of 3c05f66f0b4334a1f317161bff1a01e6 into 5c879241d007403dbb24d92524c20bb8(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:14:51,630 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3c05f66f0b4334a1f317161bff1a01e6: 2024-11-18T20:14:51,631 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., storeName=3c05f66f0b4334a1f317161bff1a01e6/A, priority=13, startTime=1731960891574; duration=0sec 2024-11-18T20:14:51,631 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T20:14:51,631 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3c05f66f0b4334a1f317161bff1a01e6:A 2024-11-18T20:14:51,631 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T20:14:51,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,632 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37149 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T20:14:51,633 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HStore(1541): 3c05f66f0b4334a1f317161bff1a01e6/C is initiating minor compaction (all files) 2024-11-18T20:14:51,633 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3c05f66f0b4334a1f317161bff1a01e6/C in TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6. 2024-11-18T20:14:51,633 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/C/1d011fd751f24e418bcf2b62a11a0577, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/C/d64d9e3da2854481be6e22007a19930c, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/C/ee73ca5a49cd4061aa515d3d7ff8dfb9] into tmpdir=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/.tmp, totalSize=36.3 K 2024-11-18T20:14:51,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,633 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1d011fd751f24e418bcf2b62a11a0577, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=240, earliestPutTs=1731960886310 2024-11-18T20:14:51,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,633 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] compactions.Compactor(225): Compacting d64d9e3da2854481be6e22007a19930c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=253, earliestPutTs=1731960887461 2024-11-18T20:14:51,633 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] compactions.Compactor(225): Compacting ee73ca5a49cd4061aa515d3d7ff8dfb9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=281, earliestPutTs=1731960889659 2024-11-18T20:14:51,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,638 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35963 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-11-18T20:14:51,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6. 2024-11-18T20:14:51,638 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3c05f66f0b4334a1f317161bff1a01e6#C#compaction#541 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:14:51,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,639 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2902): Flushing 3c05f66f0b4334a1f317161bff1a01e6 3/3 column families, dataSize=46.96 KB heapSize=123.80 KB 2024-11-18T20:14:51,639 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/.tmp/C/25fd32b82ba34f769da37d5631ab8a5d is 50, key is test_row_0/C:col10/1731960890314/Put/seqid=0 2024-11-18T20:14:51,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,646 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411181695edeb131a4f6c967fa05ab4035167_3c05f66f0b4334a1f317161bff1a01e6 is 50, key is test_row_0/A:col10/1731960890328/Put/seqid=0 2024-11-18T20:14:51,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,648 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742465_1641 (size=12949) 2024-11-18T20:14:51,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,657 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742466_1642 (size=9914) 2024-11-18T20:14:51,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:51,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-18T20:14:52,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,015 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/.tmp/B/f779191614384c52bc2fb5173a40f67d as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/B/f779191614384c52bc2fb5173a40f67d 2024-11-18T20:14:52,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,020 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3c05f66f0b4334a1f317161bff1a01e6/B of 3c05f66f0b4334a1f317161bff1a01e6 into f779191614384c52bc2fb5173a40f67d(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:14:52,020 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3c05f66f0b4334a1f317161bff1a01e6: 2024-11-18T20:14:52,020 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., storeName=3c05f66f0b4334a1f317161bff1a01e6/B, priority=13, startTime=1731960891574; duration=0sec 2024-11-18T20:14:52,020 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:14:52,020 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3c05f66f0b4334a1f317161bff1a01e6:B 2024-11-18T20:14:52,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,053 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/.tmp/C/25fd32b82ba34f769da37d5631ab8a5d as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/C/25fd32b82ba34f769da37d5631ab8a5d 2024-11-18T20:14:52,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,055 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,058 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3c05f66f0b4334a1f317161bff1a01e6/C of 3c05f66f0b4334a1f317161bff1a01e6 into 25fd32b82ba34f769da37d5631ab8a5d(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:14:52,058 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3c05f66f0b4334a1f317161bff1a01e6: 2024-11-18T20:14:52,058 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., storeName=3c05f66f0b4334a1f317161bff1a01e6/C, priority=13, startTime=1731960891574; duration=0sec 2024-11-18T20:14:52,058 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:14:52,058 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3c05f66f0b4334a1f317161bff1a01e6:C 2024-11-18T20:14:52,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,061 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411181695edeb131a4f6c967fa05ab4035167_3c05f66f0b4334a1f317161bff1a01e6 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411181695edeb131a4f6c967fa05ab4035167_3c05f66f0b4334a1f317161bff1a01e6 2024-11-18T20:14:52,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,061 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/.tmp/A/8ab88a39597e4516890a7d136445cff8, store: [table=TestAcidGuarantees family=A region=3c05f66f0b4334a1f317161bff1a01e6] 2024-11-18T20:14:52,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/.tmp/A/8ab88a39597e4516890a7d136445cff8 is 175, key is test_row_0/A:col10/1731960890328/Put/seqid=0 2024-11-18T20:14:52,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,066 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742467_1643 (size=22561) 2024-11-18T20:14:52,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,467 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=292, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/.tmp/A/8ab88a39597e4516890a7d136445cff8 2024-11-18T20:14:52,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,476 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/.tmp/B/478fdddd713b458babd6a958e0d2e3c7 is 50, key is test_row_0/B:col10/1731960890328/Put/seqid=0 2024-11-18T20:14:52,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,482 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742468_1644 (size=9857) 2024-11-18T20:14:52,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(8855): Flush requested on 3c05f66f0b4334a1f317161bff1a01e6 2024-11-18T20:14:52,497 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6. as already flushing 2024-11-18T20:14:52,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:52,536 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:52,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51828 deadline: 1731960952530, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:52,537 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:52,537 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:52,537 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:52,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51830 deadline: 1731960952533, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:52,537 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:14:52,537 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:52,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51816 deadline: 1731960952535, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:52,537 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:52,537 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:52,537 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:52,537 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:14:52,537 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:52,537 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:52,537 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:14:52,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51886 deadline: 1731960952536, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:52,538 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:52,538 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:52,538 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:14:52,545 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:52,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51868 deadline: 1731960952545, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:52,546 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:52,546 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:52,546 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:14:52,643 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:52,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51830 deadline: 1731960952643, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:52,643 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:52,644 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:52,644 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:14:52,644 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:52,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51816 deadline: 1731960952643, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:52,645 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:52,644 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:52,645 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:52,645 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:14:52,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51828 deadline: 1731960952643, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:52,645 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:52,645 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:52,645 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:14:52,645 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:52,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51886 deadline: 1731960952643, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:52,645 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:52,645 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:52,645 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:14:52,653 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:52,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51868 deadline: 1731960952653, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:52,654 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:52,654 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:52,654 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:14:52,856 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:52,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51830 deadline: 1731960952853, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:52,856 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:52,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51828 deadline: 1731960952853, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:52,856 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:52,856 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:52,856 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:52,856 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:52,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51816 deadline: 1731960952853, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:52,856 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:14:52,856 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:52,856 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:52,856 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:14:52,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51886 deadline: 1731960952853, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:52,856 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:52,856 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:52,856 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:14:52,857 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:52,857 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:52,857 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:14:52,863 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:52,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51868 deadline: 1731960952863, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:52,863 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:52,863 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:52,864 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:14:52,883 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=292 (bloomFilter=true), to=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/.tmp/B/478fdddd713b458babd6a958e0d2e3c7 2024-11-18T20:14:52,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/.tmp/C/43f60af7dabd449b84dc21250ad0cfaa is 50, key is test_row_0/C:col10/1731960890328/Put/seqid=0 2024-11-18T20:14:52,895 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742469_1645 (size=9857) 2024-11-18T20:14:53,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-18T20:14:53,165 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:53,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51830 deadline: 1731960953163, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:53,165 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:53,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51828 deadline: 1731960953163, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:53,165 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:53,166 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:53,166 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:14:53,166 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:53,166 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:53,166 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:53,166 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:53,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51886 deadline: 1731960953163, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:53,166 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:14:53,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51816 deadline: 1731960953163, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:53,166 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:53,166 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:53,166 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:53,166 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:53,166 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:14:53,166 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:14:53,173 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:53,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51868 deadline: 1731960953173, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:53,174 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:53,174 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:53,174 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:14:53,296 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=292 (bloomFilter=true), to=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/.tmp/C/43f60af7dabd449b84dc21250ad0cfaa 2024-11-18T20:14:53,299 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/.tmp/A/8ab88a39597e4516890a7d136445cff8 as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/A/8ab88a39597e4516890a7d136445cff8 2024-11-18T20:14:53,302 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/A/8ab88a39597e4516890a7d136445cff8, entries=100, sequenceid=292, filesize=22.0 K 2024-11-18T20:14:53,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/.tmp/B/478fdddd713b458babd6a958e0d2e3c7 as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/B/478fdddd713b458babd6a958e0d2e3c7 2024-11-18T20:14:53,304 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/B/478fdddd713b458babd6a958e0d2e3c7, entries=100, sequenceid=292, filesize=9.6 K 2024-11-18T20:14:53,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/.tmp/C/43f60af7dabd449b84dc21250ad0cfaa as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/C/43f60af7dabd449b84dc21250ad0cfaa 2024-11-18T20:14:53,307 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/C/43f60af7dabd449b84dc21250ad0cfaa, entries=100, sequenceid=292, filesize=9.6 K 2024-11-18T20:14:53,308 INFO [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~123.75 KB/126720, currentSize=154.31 KB/158010 for 3c05f66f0b4334a1f317161bff1a01e6 in 1670ms, sequenceid=292, compaction requested=false 2024-11-18T20:14:53,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2603): Flush status journal for 3c05f66f0b4334a1f317161bff1a01e6: 2024-11-18T20:14:53,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6. 2024-11-18T20:14:53,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c0a89b2656d4:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=167 2024-11-18T20:14:53,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41277 {}] master.HMaster(4169): Remote procedure done, pid=167 2024-11-18T20:14:53,311 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=167, resume processing ppid=166 2024-11-18T20:14:53,311 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=167, ppid=166, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.4330 sec 2024-11-18T20:14:53,312 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=166, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=166, table=TestAcidGuarantees in 2.4370 sec 2024-11-18T20:14:53,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(8855): Flush requested on 3c05f66f0b4334a1f317161bff1a01e6 2024-11-18T20:14:53,676 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-18T20:14:53,676 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3c05f66f0b4334a1f317161bff1a01e6 3/3 column families, dataSize=161.02 KB heapSize=422.63 KB 2024-11-18T20:14:53,682 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:53,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51830 deadline: 1731960953681, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:53,683 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:53,683 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:53,683 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:14:53,684 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:53,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51828 deadline: 1731960953681, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:53,684 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:53,684 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:53,684 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:14:53,685 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:53,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51886 deadline: 1731960953682, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:53,685 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:53,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51816 deadline: 1731960953682, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:53,685 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:53,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51868 deadline: 1731960953683, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:53,685 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:53,685 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:53,685 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:14:53,685 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:53,686 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:53,686 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:14:53,687 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118fbeb55ef774f4095a7744c0f0e6f4279_3c05f66f0b4334a1f317161bff1a01e6 is 50, key is test_row_0/A:col10/1731960893676/Put/seqid=0 2024-11-18T20:14:53,688 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:53,688 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:53,688 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:14:53,702 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742470_1646 (size=12454) 2024-11-18T20:14:53,794 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:53,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51830 deadline: 1731960953793, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:53,794 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:53,794 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:53,794 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:14:53,794 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:53,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51828 deadline: 1731960953793, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:53,795 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:53,795 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:53,795 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:53,795 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:14:53,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51816 deadline: 1731960953793, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:53,795 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:53,795 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:53,795 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:53,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51886 deadline: 1731960953793, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:53,795 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:14:53,795 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:53,795 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:53,795 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:14:53,939 INFO [Thread-2511 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2511. 2024-11-18T20:14:53,939 INFO [Thread-2503 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2503. 2024-11-18T20:14:53,939 DEBUG [Thread-2511 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-18T20:14:53,939 DEBUG [Thread-2503 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-18T20:14:53,939 DEBUG [Thread-2503 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:14:53,939 DEBUG [Thread-2511 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:14:53,939 INFO [Thread-2505 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2505. 2024-11-18T20:14:53,939 DEBUG [Thread-2511 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:14:53,939 DEBUG [Thread-2503 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:14:53,939 DEBUG [Thread-2505 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-18T20:14:53,939 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T20:14:53,939 DEBUG [Thread-2505 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:14:53,939 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T20:14:53,940 DEBUG [Thread-2505 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:14:53,940 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T20:14:53,941 INFO [Thread-2509 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2509. 2024-11-18T20:14:53,941 DEBUG [Thread-2509 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-18T20:14:53,941 DEBUG [Thread-2509 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:14:53,941 DEBUG [Thread-2509 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:14:53,941 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T20:14:53,942 INFO [Thread-2507 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2507. 2024-11-18T20:14:53,942 DEBUG [Thread-2507 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-18T20:14:53,942 DEBUG [Thread-2507 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:14:53,942 DEBUG [Thread-2507 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:14:53,942 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T20:14:54,004 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:54,004 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:54,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51830 deadline: 1731960954003, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:54,004 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:54,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51828 deadline: 1731960954004, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:54,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51816 deadline: 1731960954004, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:54,005 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:54,005 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:54,005 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:54,005 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:14:54,005 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:54,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51886 deadline: 1731960954004, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:54,005 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:54,005 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:54,005 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:54,005 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:14:54,005 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:14:54,005 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:54,006 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:54,006 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:14:54,104 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:54,112 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118fbeb55ef774f4095a7744c0f0e6f4279_3c05f66f0b4334a1f317161bff1a01e6 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118fbeb55ef774f4095a7744c0f0e6f4279_3c05f66f0b4334a1f317161bff1a01e6 2024-11-18T20:14:54,114 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/.tmp/A/3cfd383b3e83424bad0005919a86c28a, store: [table=TestAcidGuarantees family=A region=3c05f66f0b4334a1f317161bff1a01e6] 2024-11-18T20:14:54,114 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/.tmp/A/3cfd383b3e83424bad0005919a86c28a is 175, key is test_row_0/A:col10/1731960893676/Put/seqid=0 2024-11-18T20:14:54,117 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742471_1647 (size=31255) 2024-11-18T20:14:54,314 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:54,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51830 deadline: 1731960954314, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:54,315 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:54,315 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:54,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51816 deadline: 1731960954314, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:54,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51828 deadline: 1731960954314, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:54,316 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:54,316 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:54,316 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:54,316 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:14:54,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51886 deadline: 1731960954315, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:54,316 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:54,316 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:54,316 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:54,316 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:14:54,316 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:54,316 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:14:54,316 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:54,317 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:54,317 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:14:54,519 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=323, memsize=58.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/.tmp/A/3cfd383b3e83424bad0005919a86c28a 2024-11-18T20:14:54,530 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/.tmp/B/af7c38a084cd419881624a5e11e96d9c is 50, key is test_row_0/B:col10/1731960893676/Put/seqid=0 2024-11-18T20:14:54,533 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742472_1648 (size=12301) 2024-11-18T20:14:54,704 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:54,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51868 deadline: 1731960954704, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:54,706 WARN [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRpcRetryingCaller(177): Call to c0a89b2656d4,35963,1731960718223 for 'test_row_1' in 3c05f66f0b4334a1f317161bff1a01e6 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2160 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-18T20:14:54,707 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:54,707 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:54,707 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:14:54,823 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:54,824 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:54,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51830 deadline: 1731960954823, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:54,824 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:54,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51816 deadline: 1731960954823, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:54,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51828 deadline: 1731960954823, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:54,824 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T20:14:54,824 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:54,824 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:54,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35963 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51886 deadline: 1731960954824, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:54,824 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:14:54,824 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:54,824 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:54,824 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:54,824 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:54,824 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:14:54,824 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:14:54,824 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 , the old value is region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:54,825 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T20:14:54,825 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., hostname=c0a89b2656d4,35963,1731960718223, seqNum=5 because the exception is null or not the one we care about 2024-11-18T20:14:54,934 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58.14 KB at sequenceid=323 (bloomFilter=true), to=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/.tmp/B/af7c38a084cd419881624a5e11e96d9c 2024-11-18T20:14:54,948 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/.tmp/C/310789de67e842a3933ad57b268d0d80 is 50, key is test_row_0/C:col10/1731960893676/Put/seqid=0 2024-11-18T20:14:54,953 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742473_1649 (size=12301) 2024-11-18T20:14:55,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-18T20:14:55,045 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-18T20:14:55,354 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58.14 KB at sequenceid=323 (bloomFilter=true), to=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/.tmp/C/310789de67e842a3933ad57b268d0d80 2024-11-18T20:14:55,359 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/.tmp/A/3cfd383b3e83424bad0005919a86c28a as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/A/3cfd383b3e83424bad0005919a86c28a 2024-11-18T20:14:55,364 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/A/3cfd383b3e83424bad0005919a86c28a, entries=150, sequenceid=323, filesize=30.5 K 2024-11-18T20:14:55,365 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/.tmp/B/af7c38a084cd419881624a5e11e96d9c as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/B/af7c38a084cd419881624a5e11e96d9c 2024-11-18T20:14:55,370 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/B/af7c38a084cd419881624a5e11e96d9c, entries=150, sequenceid=323, filesize=12.0 K 2024-11-18T20:14:55,371 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/.tmp/C/310789de67e842a3933ad57b268d0d80 as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/C/310789de67e842a3933ad57b268d0d80 2024-11-18T20:14:55,375 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/C/310789de67e842a3933ad57b268d0d80, entries=150, sequenceid=323, filesize=12.0 K 2024-11-18T20:14:55,376 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~174.43 KB/178620, heapSize ~457.73 KB/468720, currentSize=33.54 KB/34350 for 3c05f66f0b4334a1f317161bff1a01e6 in 1700ms, sequenceid=323, compaction requested=true 2024-11-18T20:14:55,376 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3c05f66f0b4334a1f317161bff1a01e6: 2024-11-18T20:14:55,376 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3c05f66f0b4334a1f317161bff1a01e6:A, priority=-2147483648, current under compaction store size is 1 2024-11-18T20:14:55,376 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:14:55,376 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3c05f66f0b4334a1f317161bff1a01e6:B, priority=-2147483648, current under compaction store size is 2 2024-11-18T20:14:55,376 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T20:14:55,377 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:14:55,377 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3c05f66f0b4334a1f317161bff1a01e6:C, priority=-2147483648, current under compaction store size is 3 2024-11-18T20:14:55,377 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T20:14:55,377 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T20:14:55,378 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 85719 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T20:14:55,378 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HStore(1541): 3c05f66f0b4334a1f317161bff1a01e6/A is initiating minor compaction (all files) 2024-11-18T20:14:55,378 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35107 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T20:14:55,378 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3c05f66f0b4334a1f317161bff1a01e6/A in TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6. 2024-11-18T20:14:55,378 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HStore(1541): 3c05f66f0b4334a1f317161bff1a01e6/B is initiating minor compaction (all files) 2024-11-18T20:14:55,378 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/A/5c879241d007403dbb24d92524c20bb8, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/A/8ab88a39597e4516890a7d136445cff8, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/A/3cfd383b3e83424bad0005919a86c28a] into tmpdir=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/.tmp, totalSize=83.7 K 2024-11-18T20:14:55,378 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3c05f66f0b4334a1f317161bff1a01e6/B in TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6. 2024-11-18T20:14:55,378 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6. 2024-11-18T20:14:55,378 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/B/f779191614384c52bc2fb5173a40f67d, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/B/478fdddd713b458babd6a958e0d2e3c7, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/B/af7c38a084cd419881624a5e11e96d9c] into tmpdir=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/.tmp, totalSize=34.3 K 2024-11-18T20:14:55,378 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6. files: [hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/A/5c879241d007403dbb24d92524c20bb8, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/A/8ab88a39597e4516890a7d136445cff8, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/A/3cfd383b3e83424bad0005919a86c28a] 2024-11-18T20:14:55,378 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5c879241d007403dbb24d92524c20bb8, keycount=150, bloomtype=ROW, size=31.2 K, encoding=NONE, compression=NONE, seqNum=281, earliestPutTs=1731960889659 2024-11-18T20:14:55,378 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.Compactor(225): Compacting f779191614384c52bc2fb5173a40f67d, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=281, earliestPutTs=1731960889659 2024-11-18T20:14:55,379 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.Compactor(225): Compacting 478fdddd713b458babd6a958e0d2e3c7, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=292, earliestPutTs=1731960890328 2024-11-18T20:14:55,379 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8ab88a39597e4516890a7d136445cff8, keycount=100, bloomtype=ROW, size=22.0 K, encoding=NONE, compression=NONE, seqNum=292, earliestPutTs=1731960890328 2024-11-18T20:14:55,379 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.Compactor(225): Compacting af7c38a084cd419881624a5e11e96d9c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=323, earliestPutTs=1731960892534 2024-11-18T20:14:55,379 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3cfd383b3e83424bad0005919a86c28a, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=323, earliestPutTs=1731960892534 2024-11-18T20:14:55,387 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=3c05f66f0b4334a1f317161bff1a01e6] 2024-11-18T20:14:55,388 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3c05f66f0b4334a1f317161bff1a01e6#B#compaction#548 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:14:55,389 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/.tmp/B/5a965c51b4254c91bbf67acc26f0e269 is 50, key is test_row_0/B:col10/1731960893676/Put/seqid=0 2024-11-18T20:14:55,389 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241118c84ba0f0957044378772ce8581d6c008_3c05f66f0b4334a1f317161bff1a01e6 store=[table=TestAcidGuarantees family=A region=3c05f66f0b4334a1f317161bff1a01e6] 2024-11-18T20:14:55,392 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241118c84ba0f0957044378772ce8581d6c008_3c05f66f0b4334a1f317161bff1a01e6, store=[table=TestAcidGuarantees family=A region=3c05f66f0b4334a1f317161bff1a01e6] 2024-11-18T20:14:55,392 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118c84ba0f0957044378772ce8581d6c008_3c05f66f0b4334a1f317161bff1a01e6 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=3c05f66f0b4334a1f317161bff1a01e6] 2024-11-18T20:14:55,392 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742474_1650 (size=13051) 2024-11-18T20:14:55,394 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742475_1651 (size=4469) 2024-11-18T20:14:55,798 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3c05f66f0b4334a1f317161bff1a01e6#A#compaction#549 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:14:55,799 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/.tmp/A/d520d1f3249443188151a69ed8d51f62 is 175, key is test_row_0/A:col10/1731960893676/Put/seqid=0 2024-11-18T20:14:55,803 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/.tmp/B/5a965c51b4254c91bbf67acc26f0e269 as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/B/5a965c51b4254c91bbf67acc26f0e269 2024-11-18T20:14:55,803 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742476_1652 (size=32005) 2024-11-18T20:14:55,807 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3c05f66f0b4334a1f317161bff1a01e6/B of 3c05f66f0b4334a1f317161bff1a01e6 into 5a965c51b4254c91bbf67acc26f0e269(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:14:55,807 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3c05f66f0b4334a1f317161bff1a01e6: 2024-11-18T20:14:55,807 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., storeName=3c05f66f0b4334a1f317161bff1a01e6/B, priority=13, startTime=1731960895376; duration=0sec 2024-11-18T20:14:55,807 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T20:14:55,807 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3c05f66f0b4334a1f317161bff1a01e6:B 2024-11-18T20:14:55,807 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T20:14:55,808 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35107 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T20:14:55,808 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HStore(1541): 3c05f66f0b4334a1f317161bff1a01e6/C is initiating minor compaction (all files) 2024-11-18T20:14:55,808 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3c05f66f0b4334a1f317161bff1a01e6/C in TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6. 2024-11-18T20:14:55,808 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/C/25fd32b82ba34f769da37d5631ab8a5d, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/C/43f60af7dabd449b84dc21250ad0cfaa, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/C/310789de67e842a3933ad57b268d0d80] into tmpdir=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/.tmp, totalSize=34.3 K 2024-11-18T20:14:55,808 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.Compactor(225): Compacting 25fd32b82ba34f769da37d5631ab8a5d, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=281, earliestPutTs=1731960889659 2024-11-18T20:14:55,809 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.Compactor(225): Compacting 43f60af7dabd449b84dc21250ad0cfaa, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=292, earliestPutTs=1731960890328 2024-11-18T20:14:55,809 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] compactions.Compactor(225): Compacting 310789de67e842a3933ad57b268d0d80, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=323, earliestPutTs=1731960892534 2024-11-18T20:14:55,815 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3c05f66f0b4334a1f317161bff1a01e6#C#compaction#550 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T20:14:55,815 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/.tmp/C/8e6dceeaa94f446cbe35060ef80d78b8 is 50, key is test_row_0/C:col10/1731960893676/Put/seqid=0 2024-11-18T20:14:55,818 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742477_1653 (size=13051) 2024-11-18T20:14:55,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35963 {}] regionserver.HRegion(8855): Flush requested on 3c05f66f0b4334a1f317161bff1a01e6 2024-11-18T20:14:55,836 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-18T20:14:55,836 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3c05f66f0b4334a1f317161bff1a01e6 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-18T20:14:55,836 INFO [Thread-2492 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2492. 2024-11-18T20:14:55,836 INFO [Thread-2494 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2494. 2024-11-18T20:14:55,836 DEBUG [Thread-2492 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-18T20:14:55,836 INFO [Thread-2500 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2500. 2024-11-18T20:14:55,836 DEBUG [Thread-2492 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:14:55,836 DEBUG [Thread-2494 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-18T20:14:55,836 DEBUG [Thread-2500 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-18T20:14:55,836 DEBUG [Thread-2494 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:14:55,836 DEBUG [Thread-2500 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:14:55,836 DEBUG [Thread-2492 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:14:55,836 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T20:14:55,836 DEBUG [Thread-2500 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:14:55,836 DEBUG [Thread-2494 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:14:55,837 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T20:14:55,837 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T20:14:55,843 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111859a165163e0749388b803fffbdcb6a8d_3c05f66f0b4334a1f317161bff1a01e6 is 50, key is test_row_0/A:col10/1731960895834/Put/seqid=0 2024-11-18T20:14:55,844 INFO [Thread-2496 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2496. 2024-11-18T20:14:55,844 DEBUG [Thread-2496 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-18T20:14:55,845 DEBUG [Thread-2496 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:14:55,845 DEBUG [Thread-2496 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:14:55,845 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T20:14:55,846 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742478_1654 (size=12454) 2024-11-18T20:14:56,214 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/.tmp/A/d520d1f3249443188151a69ed8d51f62 as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/A/d520d1f3249443188151a69ed8d51f62 2024-11-18T20:14:56,220 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3c05f66f0b4334a1f317161bff1a01e6/A of 3c05f66f0b4334a1f317161bff1a01e6 into d520d1f3249443188151a69ed8d51f62(size=31.3 K), total size for store is 31.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:14:56,220 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3c05f66f0b4334a1f317161bff1a01e6: 2024-11-18T20:14:56,220 INFO [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., storeName=3c05f66f0b4334a1f317161bff1a01e6/A, priority=13, startTime=1731960895376; duration=0sec 2024-11-18T20:14:56,221 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:14:56,221 DEBUG [RS:0;c0a89b2656d4:35963-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3c05f66f0b4334a1f317161bff1a01e6:A 2024-11-18T20:14:56,223 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/.tmp/C/8e6dceeaa94f446cbe35060ef80d78b8 as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/C/8e6dceeaa94f446cbe35060ef80d78b8 2024-11-18T20:14:56,226 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3c05f66f0b4334a1f317161bff1a01e6/C of 3c05f66f0b4334a1f317161bff1a01e6 into 8e6dceeaa94f446cbe35060ef80d78b8(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T20:14:56,226 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3c05f66f0b4334a1f317161bff1a01e6: 2024-11-18T20:14:56,226 INFO [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., storeName=3c05f66f0b4334a1f317161bff1a01e6/C, priority=13, startTime=1731960895377; duration=0sec 2024-11-18T20:14:56,226 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T20:14:56,226 DEBUG [RS:0;c0a89b2656d4:35963-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3c05f66f0b4334a1f317161bff1a01e6:C 2024-11-18T20:14:56,247 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:56,251 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111859a165163e0749388b803fffbdcb6a8d_3c05f66f0b4334a1f317161bff1a01e6 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111859a165163e0749388b803fffbdcb6a8d_3c05f66f0b4334a1f317161bff1a01e6 2024-11-18T20:14:56,252 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/.tmp/A/aafdcab304b44c0f81be4a529a01bd96, store: [table=TestAcidGuarantees family=A region=3c05f66f0b4334a1f317161bff1a01e6] 2024-11-18T20:14:56,252 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/.tmp/A/aafdcab304b44c0f81be4a529a01bd96 is 175, key is test_row_0/A:col10/1731960895834/Put/seqid=0 2024-11-18T20:14:56,257 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742479_1655 (size=31255) 2024-11-18T20:14:56,535 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-18T20:14:56,658 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=335, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/.tmp/A/aafdcab304b44c0f81be4a529a01bd96 2024-11-18T20:14:56,669 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/.tmp/B/c88191c21b844899b8a17fcc37461cf3 is 50, key is test_row_0/B:col10/1731960895834/Put/seqid=0 2024-11-18T20:14:56,674 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742480_1656 (size=12301) 2024-11-18T20:14:56,724 INFO [Thread-2498 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2498. 2024-11-18T20:14:56,724 DEBUG [Thread-2498 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-18T20:14:56,724 DEBUG [Thread-2498 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:14:56,725 DEBUG [Thread-2498 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:14:56,725 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T20:14:56,725 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-18T20:14:56,725 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 60 2024-11-18T20:14:56,725 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 47 2024-11-18T20:14:56,725 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 57 2024-11-18T20:14:56,725 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 47 2024-11-18T20:14:56,725 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 52 2024-11-18T20:14:56,725 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-18T20:14:56,725 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4175 2024-11-18T20:14:56,725 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4220 2024-11-18T20:14:56,725 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4227 2024-11-18T20:14:56,725 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4176 2024-11-18T20:14:56,725 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4236 2024-11-18T20:14:56,725 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-18T20:14:56,725 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-18T20:14:56,725 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobGetAtomicity(AcidGuaranteesTestBase.java:122) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-18T20:14:56,725 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:14:56,725 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:14:56,725 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T20:14:56,726 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41277 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-18T20:14:56,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41277 {}] procedure2.ProcedureExecutor(1139): Stored pid=168, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-18T20:14:56,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-18T20:14:56,728 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731960896728"}]},"ts":"1731960896728"} 2024-11-18T20:14:56,729 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-18T20:14:56,729 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-18T20:14:56,730 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=169, ppid=168, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-18T20:14:56,731 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=170, ppid=169, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=3c05f66f0b4334a1f317161bff1a01e6, UNASSIGN}] 2024-11-18T20:14:56,732 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=170, ppid=169, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=3c05f66f0b4334a1f317161bff1a01e6, UNASSIGN 2024-11-18T20:14:56,733 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=170 updating hbase:meta row=3c05f66f0b4334a1f317161bff1a01e6, regionState=CLOSING, regionLocation=c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:56,734 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=170, ppid=169, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=3c05f66f0b4334a1f317161bff1a01e6, UNASSIGN because future has completed 2024-11-18T20:14:56,734 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-18T20:14:56,734 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=171, ppid=170, state=RUNNABLE, hasLock=false; CloseRegionProcedure 3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223}] 2024-11-18T20:14:56,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-18T20:14:56,887 INFO [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] handler.UnassignRegionHandler(122): Close 3c05f66f0b4334a1f317161bff1a01e6 2024-11-18T20:14:56,887 DEBUG [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-18T20:14:56,887 DEBUG [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.HRegion(1722): Closing 3c05f66f0b4334a1f317161bff1a01e6, disabling compactions & flushes 2024-11-18T20:14:56,887 DEBUG [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.HRegion(1993): waiting for 0 compactions & cache flush to complete for region TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6. 2024-11-18T20:14:57,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-18T20:14:57,076 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=335 (bloomFilter=true), to=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/.tmp/B/c88191c21b844899b8a17fcc37461cf3 2024-11-18T20:14:57,089 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/.tmp/C/e1f1fa7bc2f146dfb3adf82072ea4dff is 50, key is test_row_0/C:col10/1731960895834/Put/seqid=0 2024-11-18T20:14:57,093 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742481_1657 (size=12301) 2024-11-18T20:14:57,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-18T20:14:57,493 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=335 (bloomFilter=true), to=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/.tmp/C/e1f1fa7bc2f146dfb3adf82072ea4dff 2024-11-18T20:14:57,496 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/.tmp/A/aafdcab304b44c0f81be4a529a01bd96 as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/A/aafdcab304b44c0f81be4a529a01bd96 2024-11-18T20:14:57,498 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/A/aafdcab304b44c0f81be4a529a01bd96, entries=150, sequenceid=335, filesize=30.5 K 2024-11-18T20:14:57,499 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/.tmp/B/c88191c21b844899b8a17fcc37461cf3 as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/B/c88191c21b844899b8a17fcc37461cf3 2024-11-18T20:14:57,501 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/B/c88191c21b844899b8a17fcc37461cf3, entries=150, sequenceid=335, filesize=12.0 K 2024-11-18T20:14:57,501 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/.tmp/C/e1f1fa7bc2f146dfb3adf82072ea4dff as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/C/e1f1fa7bc2f146dfb3adf82072ea4dff 2024-11-18T20:14:57,504 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/C/e1f1fa7bc2f146dfb3adf82072ea4dff, entries=150, sequenceid=335, filesize=12.0 K 2024-11-18T20:14:57,505 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=13.42 KB/13740 for 3c05f66f0b4334a1f317161bff1a01e6 in 1668ms, sequenceid=335, compaction requested=false 2024-11-18T20:14:57,505 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3c05f66f0b4334a1f317161bff1a01e6: 2024-11-18T20:14:57,505 INFO [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6. 2024-11-18T20:14:57,505 DEBUG [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6. 2024-11-18T20:14:57,505 DEBUG [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6. after waiting 0 ms 2024-11-18T20:14:57,505 DEBUG [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6. 2024-11-18T20:14:57,505 INFO [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.HRegion(2902): Flushing 3c05f66f0b4334a1f317161bff1a01e6 3/3 column families, dataSize=13.42 KB heapSize=35.91 KB 2024-11-18T20:14:57,509 DEBUG [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111891224a0705e44ecc9272b377bcc90411_3c05f66f0b4334a1f317161bff1a01e6 is 50, key is test_row_1/A:col10/1731960896723/Put/seqid=0 2024-11-18T20:14:57,511 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742482_1658 (size=9914) 2024-11-18T20:14:57,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-18T20:14:57,912 DEBUG [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T20:14:57,918 INFO [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111891224a0705e44ecc9272b377bcc90411_3c05f66f0b4334a1f317161bff1a01e6 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111891224a0705e44ecc9272b377bcc90411_3c05f66f0b4334a1f317161bff1a01e6 2024-11-18T20:14:57,919 DEBUG [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/.tmp/A/7611b6074be54f169d101bac5159f863, store: [table=TestAcidGuarantees family=A region=3c05f66f0b4334a1f317161bff1a01e6] 2024-11-18T20:14:57,920 DEBUG [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/.tmp/A/7611b6074be54f169d101bac5159f863 is 175, key is test_row_1/A:col10/1731960896723/Put/seqid=0 2024-11-18T20:14:57,925 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742483_1659 (size=22561) 2024-11-18T20:14:58,326 INFO [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=342, memsize=4.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/.tmp/A/7611b6074be54f169d101bac5159f863 2024-11-18T20:14:58,333 DEBUG [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/.tmp/B/a1c59ea701d14cf7ab841de1e776b2a3 is 50, key is test_row_1/B:col10/1731960896723/Put/seqid=0 2024-11-18T20:14:58,337 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742484_1660 (size=9857) 2024-11-18T20:14:58,338 INFO [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=342 (bloomFilter=true), to=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/.tmp/B/a1c59ea701d14cf7ab841de1e776b2a3 2024-11-18T20:14:58,343 DEBUG [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/.tmp/C/ef6fe427162c4a88a997bf2e018c6e4f is 50, key is test_row_1/C:col10/1731960896723/Put/seqid=0 2024-11-18T20:14:58,346 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742485_1661 (size=9857) 2024-11-18T20:14:58,748 INFO [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=342 (bloomFilter=true), to=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/.tmp/C/ef6fe427162c4a88a997bf2e018c6e4f 2024-11-18T20:14:58,757 DEBUG [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/.tmp/A/7611b6074be54f169d101bac5159f863 as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/A/7611b6074be54f169d101bac5159f863 2024-11-18T20:14:58,761 INFO [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/A/7611b6074be54f169d101bac5159f863, entries=100, sequenceid=342, filesize=22.0 K 2024-11-18T20:14:58,762 DEBUG [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/.tmp/B/a1c59ea701d14cf7ab841de1e776b2a3 as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/B/a1c59ea701d14cf7ab841de1e776b2a3 2024-11-18T20:14:58,765 INFO [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/B/a1c59ea701d14cf7ab841de1e776b2a3, entries=100, sequenceid=342, filesize=9.6 K 2024-11-18T20:14:58,765 DEBUG [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/.tmp/C/ef6fe427162c4a88a997bf2e018c6e4f as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/C/ef6fe427162c4a88a997bf2e018c6e4f 2024-11-18T20:14:58,767 INFO [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/C/ef6fe427162c4a88a997bf2e018c6e4f, entries=100, sequenceid=342, filesize=9.6 K 2024-11-18T20:14:58,768 INFO [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.HRegion(3140): Finished flush of dataSize ~13.42 KB/13740, heapSize ~35.86 KB/36720, currentSize=0 B/0 for 3c05f66f0b4334a1f317161bff1a01e6 in 1263ms, sequenceid=342, compaction requested=true 2024-11-18T20:14:58,769 DEBUG [StoreCloser-TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/A/e3bf4d6373ad4fe4b376de9dbaee5f38, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/A/dcd8f8aaff52436385c61886be3a3dc3, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/A/c5cd93c24e68438495fb8ff6b04e83b5, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/A/825d5734db984c7aa6293f8eba885cd5, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/A/c0cf7c7af19743ee921a3326887d970e, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/A/bac0ca3c4b4f491cbdcab34ab03fec7d, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/A/68ef0ab4ff4843beaddcba61b314befb, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/A/7a4faf779614496bb5280a805576a54f, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/A/fadc8d61b3c1491986ff885366edcad8, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/A/d4c3c30b89fb41e78fbf4142ea756050, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/A/50275b6b2e1f4e1a858a21e6f6327e0b, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/A/b767b9cf07ee4754b1a3d3de58e739b7, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/A/63cf1986db11465fa711efde1833e8ae, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/A/95369b3c7711447ea70140c840e1da0f, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/A/ccd0ea0e74be44e5b46bac4695497976, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/A/93d5c4fbaa2a490fb98bca2c9671e41e, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/A/0a9708af986d4f079e03741337c2467a, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/A/626d51fb026c4c31995c3ef99d01a752, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/A/5c879241d007403dbb24d92524c20bb8, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/A/dfe351f4f0734f979741c88c2c6862bf, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/A/8ab88a39597e4516890a7d136445cff8, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/A/3cfd383b3e83424bad0005919a86c28a] to archive 2024-11-18T20:14:58,769 DEBUG [StoreCloser-TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-18T20:14:58,770 DEBUG [StoreCloser-TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/A/e3bf4d6373ad4fe4b376de9dbaee5f38 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/A/e3bf4d6373ad4fe4b376de9dbaee5f38 2024-11-18T20:14:58,771 DEBUG [StoreCloser-TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/A/dcd8f8aaff52436385c61886be3a3dc3 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/A/dcd8f8aaff52436385c61886be3a3dc3 2024-11-18T20:14:58,772 DEBUG [StoreCloser-TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/A/c5cd93c24e68438495fb8ff6b04e83b5 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/A/c5cd93c24e68438495fb8ff6b04e83b5 2024-11-18T20:14:58,773 DEBUG [StoreCloser-TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/A/825d5734db984c7aa6293f8eba885cd5 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/A/825d5734db984c7aa6293f8eba885cd5 2024-11-18T20:14:58,774 DEBUG [StoreCloser-TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/A/c0cf7c7af19743ee921a3326887d970e to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/A/c0cf7c7af19743ee921a3326887d970e 2024-11-18T20:14:58,774 DEBUG [StoreCloser-TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/A/bac0ca3c4b4f491cbdcab34ab03fec7d to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/A/bac0ca3c4b4f491cbdcab34ab03fec7d 2024-11-18T20:14:58,775 DEBUG [StoreCloser-TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/A/68ef0ab4ff4843beaddcba61b314befb to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/A/68ef0ab4ff4843beaddcba61b314befb 2024-11-18T20:14:58,776 DEBUG [StoreCloser-TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/A/7a4faf779614496bb5280a805576a54f to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/A/7a4faf779614496bb5280a805576a54f 2024-11-18T20:14:58,776 DEBUG [StoreCloser-TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/A/fadc8d61b3c1491986ff885366edcad8 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/A/fadc8d61b3c1491986ff885366edcad8 2024-11-18T20:14:58,777 DEBUG [StoreCloser-TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/A/d4c3c30b89fb41e78fbf4142ea756050 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/A/d4c3c30b89fb41e78fbf4142ea756050 2024-11-18T20:14:58,778 DEBUG [StoreCloser-TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/A/50275b6b2e1f4e1a858a21e6f6327e0b to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/A/50275b6b2e1f4e1a858a21e6f6327e0b 2024-11-18T20:14:58,778 DEBUG [StoreCloser-TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/A/b767b9cf07ee4754b1a3d3de58e739b7 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/A/b767b9cf07ee4754b1a3d3de58e739b7 2024-11-18T20:14:58,779 DEBUG [StoreCloser-TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/A/63cf1986db11465fa711efde1833e8ae to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/A/63cf1986db11465fa711efde1833e8ae 2024-11-18T20:14:58,780 DEBUG [StoreCloser-TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/A/95369b3c7711447ea70140c840e1da0f to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/A/95369b3c7711447ea70140c840e1da0f 2024-11-18T20:14:58,781 DEBUG [StoreCloser-TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/A/ccd0ea0e74be44e5b46bac4695497976 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/A/ccd0ea0e74be44e5b46bac4695497976 2024-11-18T20:14:58,781 DEBUG [StoreCloser-TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/A/93d5c4fbaa2a490fb98bca2c9671e41e to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/A/93d5c4fbaa2a490fb98bca2c9671e41e 2024-11-18T20:14:58,782 DEBUG [StoreCloser-TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/A/0a9708af986d4f079e03741337c2467a to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/A/0a9708af986d4f079e03741337c2467a 2024-11-18T20:14:58,783 DEBUG [StoreCloser-TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/A/626d51fb026c4c31995c3ef99d01a752 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/A/626d51fb026c4c31995c3ef99d01a752 2024-11-18T20:14:58,784 DEBUG [StoreCloser-TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/A/5c879241d007403dbb24d92524c20bb8 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/A/5c879241d007403dbb24d92524c20bb8 2024-11-18T20:14:58,784 DEBUG [StoreCloser-TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/A/dfe351f4f0734f979741c88c2c6862bf to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/A/dfe351f4f0734f979741c88c2c6862bf 2024-11-18T20:14:58,785 DEBUG [StoreCloser-TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/A/8ab88a39597e4516890a7d136445cff8 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/A/8ab88a39597e4516890a7d136445cff8 2024-11-18T20:14:58,785 DEBUG [StoreCloser-TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/A/3cfd383b3e83424bad0005919a86c28a to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/A/3cfd383b3e83424bad0005919a86c28a 2024-11-18T20:14:58,787 DEBUG [StoreCloser-TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/B/dde5d7090e37498ba6387e978b2b6051, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/B/92f9e0980b0e4e2ca402204d5b4f7c8e, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/B/8c89c95908234d6d84f6bf1f371176a0, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/B/1ac398a88c55448cabc84eaf0e959165, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/B/a9969320227b4298a623918c098a470f, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/B/406477df55174c6492832fb032e707c6, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/B/aa29dd98ec0749c8ae547bcc0c1e6aff, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/B/5bd7b758d4b8439d92e68727dd47e52a, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/B/7bfc490bd6514e9cb83a90b85042b717, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/B/c119b21db4d44d429f3a5461f614fd49, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/B/f5b46135f83d4485901d300da5d36d41, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/B/81fe69dbf1c5439fb9f37f6996d1d746, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/B/ecd351c7261743d3a8daaa3ceef0efb6, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/B/fbf5a743d11e41cea554aa2b3deccc57, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/B/67c934fed476488bada5d8c0951fef8f, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/B/7a492d87e2cf4580b77f9b279fa6ac00, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/B/ec9ee31cf4914217bb761a835455025a, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/B/613d0a3f582041d5a2c03665cf59e25f, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/B/f779191614384c52bc2fb5173a40f67d, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/B/a9a57ebbaf5d48afb588a9e22255bb99, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/B/478fdddd713b458babd6a958e0d2e3c7, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/B/af7c38a084cd419881624a5e11e96d9c] to archive 2024-11-18T20:14:58,787 DEBUG [StoreCloser-TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-18T20:14:58,788 DEBUG [StoreCloser-TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/B/dde5d7090e37498ba6387e978b2b6051 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/B/dde5d7090e37498ba6387e978b2b6051 2024-11-18T20:14:58,789 DEBUG [StoreCloser-TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/B/92f9e0980b0e4e2ca402204d5b4f7c8e to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/B/92f9e0980b0e4e2ca402204d5b4f7c8e 2024-11-18T20:14:58,789 DEBUG [StoreCloser-TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/B/8c89c95908234d6d84f6bf1f371176a0 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/B/8c89c95908234d6d84f6bf1f371176a0 2024-11-18T20:14:58,790 DEBUG [StoreCloser-TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/B/1ac398a88c55448cabc84eaf0e959165 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/B/1ac398a88c55448cabc84eaf0e959165 2024-11-18T20:14:58,791 DEBUG [StoreCloser-TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/B/a9969320227b4298a623918c098a470f to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/B/a9969320227b4298a623918c098a470f 2024-11-18T20:14:58,791 DEBUG [StoreCloser-TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/B/406477df55174c6492832fb032e707c6 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/B/406477df55174c6492832fb032e707c6 2024-11-18T20:14:58,792 DEBUG [StoreCloser-TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/B/aa29dd98ec0749c8ae547bcc0c1e6aff to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/B/aa29dd98ec0749c8ae547bcc0c1e6aff 2024-11-18T20:14:58,792 DEBUG [StoreCloser-TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/B/5bd7b758d4b8439d92e68727dd47e52a to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/B/5bd7b758d4b8439d92e68727dd47e52a 2024-11-18T20:14:58,793 DEBUG [StoreCloser-TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/B/7bfc490bd6514e9cb83a90b85042b717 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/B/7bfc490bd6514e9cb83a90b85042b717 2024-11-18T20:14:58,794 DEBUG [StoreCloser-TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/B/c119b21db4d44d429f3a5461f614fd49 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/B/c119b21db4d44d429f3a5461f614fd49 2024-11-18T20:14:58,794 DEBUG [StoreCloser-TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/B/f5b46135f83d4485901d300da5d36d41 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/B/f5b46135f83d4485901d300da5d36d41 2024-11-18T20:14:58,795 DEBUG [StoreCloser-TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/B/81fe69dbf1c5439fb9f37f6996d1d746 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/B/81fe69dbf1c5439fb9f37f6996d1d746 2024-11-18T20:14:58,796 DEBUG [StoreCloser-TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/B/ecd351c7261743d3a8daaa3ceef0efb6 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/B/ecd351c7261743d3a8daaa3ceef0efb6 2024-11-18T20:14:58,796 DEBUG [StoreCloser-TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/B/fbf5a743d11e41cea554aa2b3deccc57 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/B/fbf5a743d11e41cea554aa2b3deccc57 2024-11-18T20:14:58,797 DEBUG [StoreCloser-TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/B/67c934fed476488bada5d8c0951fef8f to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/B/67c934fed476488bada5d8c0951fef8f 2024-11-18T20:14:58,797 DEBUG [StoreCloser-TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/B/7a492d87e2cf4580b77f9b279fa6ac00 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/B/7a492d87e2cf4580b77f9b279fa6ac00 2024-11-18T20:14:58,798 DEBUG [StoreCloser-TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/B/ec9ee31cf4914217bb761a835455025a to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/B/ec9ee31cf4914217bb761a835455025a 2024-11-18T20:14:58,799 DEBUG [StoreCloser-TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/B/613d0a3f582041d5a2c03665cf59e25f to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/B/613d0a3f582041d5a2c03665cf59e25f 2024-11-18T20:14:58,799 DEBUG [StoreCloser-TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/B/f779191614384c52bc2fb5173a40f67d to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/B/f779191614384c52bc2fb5173a40f67d 2024-11-18T20:14:58,800 DEBUG [StoreCloser-TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/B/a9a57ebbaf5d48afb588a9e22255bb99 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/B/a9a57ebbaf5d48afb588a9e22255bb99 2024-11-18T20:14:58,801 DEBUG [StoreCloser-TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/B/478fdddd713b458babd6a958e0d2e3c7 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/B/478fdddd713b458babd6a958e0d2e3c7 2024-11-18T20:14:58,801 DEBUG [StoreCloser-TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/B/af7c38a084cd419881624a5e11e96d9c to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/B/af7c38a084cd419881624a5e11e96d9c 2024-11-18T20:14:58,802 DEBUG [StoreCloser-TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/C/b45e7b7087184f66b871753c54948e6f, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/C/b308f86f673b4f4c911dceb4b2e69145, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/C/fb61bacd061b4933b2c51aec4c23222e, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/C/6555f5af1fd54e3a93288abcfe896342, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/C/076e25d583474d2ab57a817a1aeff2d2, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/C/9075c6f7a13e4565b98ab50c0d4c270a, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/C/12fb2791886f49f5948d8a0049a0a20e, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/C/5da7863cd7b842d79f8d1203b850ccfc, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/C/a679dcf47f314857a7455151ac18552a, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/C/e5004530da284f7ebf23a6e67d708647, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/C/ec2aa7a8035344f7a9070cc3333480ed, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/C/146c39e37c1a460088e2ab1f2cb0f202, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/C/e091ead8117f4549933b792712b3f574, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/C/debde896eb0447b7a6f386d72bdc43e4, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/C/8df4e12f03b34b4ba51e40550ef94729, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/C/1d011fd751f24e418bcf2b62a11a0577, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/C/b621c19670b34fe6bf1956bf60f127e2, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/C/d64d9e3da2854481be6e22007a19930c, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/C/25fd32b82ba34f769da37d5631ab8a5d, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/C/ee73ca5a49cd4061aa515d3d7ff8dfb9, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/C/43f60af7dabd449b84dc21250ad0cfaa, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/C/310789de67e842a3933ad57b268d0d80] to archive 2024-11-18T20:14:58,803 DEBUG [StoreCloser-TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-18T20:14:58,804 DEBUG [StoreCloser-TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/C/b45e7b7087184f66b871753c54948e6f to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/C/b45e7b7087184f66b871753c54948e6f 2024-11-18T20:14:58,804 DEBUG [StoreCloser-TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/C/b308f86f673b4f4c911dceb4b2e69145 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/C/b308f86f673b4f4c911dceb4b2e69145 2024-11-18T20:14:58,805 DEBUG [StoreCloser-TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/C/fb61bacd061b4933b2c51aec4c23222e to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/C/fb61bacd061b4933b2c51aec4c23222e 2024-11-18T20:14:58,806 DEBUG [StoreCloser-TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/C/6555f5af1fd54e3a93288abcfe896342 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/C/6555f5af1fd54e3a93288abcfe896342 2024-11-18T20:14:58,807 DEBUG [StoreCloser-TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/C/076e25d583474d2ab57a817a1aeff2d2 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/C/076e25d583474d2ab57a817a1aeff2d2 2024-11-18T20:14:58,807 DEBUG [StoreCloser-TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/C/9075c6f7a13e4565b98ab50c0d4c270a to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/C/9075c6f7a13e4565b98ab50c0d4c270a 2024-11-18T20:14:58,808 DEBUG [StoreCloser-TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/C/12fb2791886f49f5948d8a0049a0a20e to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/C/12fb2791886f49f5948d8a0049a0a20e 2024-11-18T20:14:58,808 DEBUG [StoreCloser-TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/C/5da7863cd7b842d79f8d1203b850ccfc to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/C/5da7863cd7b842d79f8d1203b850ccfc 2024-11-18T20:14:58,809 DEBUG [StoreCloser-TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/C/a679dcf47f314857a7455151ac18552a to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/C/a679dcf47f314857a7455151ac18552a 2024-11-18T20:14:58,810 DEBUG [StoreCloser-TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/C/e5004530da284f7ebf23a6e67d708647 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/C/e5004530da284f7ebf23a6e67d708647 2024-11-18T20:14:58,810 DEBUG [StoreCloser-TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/C/ec2aa7a8035344f7a9070cc3333480ed to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/C/ec2aa7a8035344f7a9070cc3333480ed 2024-11-18T20:14:58,811 DEBUG [StoreCloser-TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/C/146c39e37c1a460088e2ab1f2cb0f202 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/C/146c39e37c1a460088e2ab1f2cb0f202 2024-11-18T20:14:58,811 DEBUG [StoreCloser-TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/C/e091ead8117f4549933b792712b3f574 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/C/e091ead8117f4549933b792712b3f574 2024-11-18T20:14:58,812 DEBUG [StoreCloser-TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/C/debde896eb0447b7a6f386d72bdc43e4 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/C/debde896eb0447b7a6f386d72bdc43e4 2024-11-18T20:14:58,813 DEBUG [StoreCloser-TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/C/8df4e12f03b34b4ba51e40550ef94729 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/C/8df4e12f03b34b4ba51e40550ef94729 2024-11-18T20:14:58,813 DEBUG [StoreCloser-TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/C/1d011fd751f24e418bcf2b62a11a0577 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/C/1d011fd751f24e418bcf2b62a11a0577 2024-11-18T20:14:58,814 DEBUG [StoreCloser-TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/C/b621c19670b34fe6bf1956bf60f127e2 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/C/b621c19670b34fe6bf1956bf60f127e2 2024-11-18T20:14:58,814 DEBUG [StoreCloser-TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/C/d64d9e3da2854481be6e22007a19930c to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/C/d64d9e3da2854481be6e22007a19930c 2024-11-18T20:14:58,815 DEBUG [StoreCloser-TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/C/25fd32b82ba34f769da37d5631ab8a5d to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/C/25fd32b82ba34f769da37d5631ab8a5d 2024-11-18T20:14:58,816 DEBUG [StoreCloser-TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/C/ee73ca5a49cd4061aa515d3d7ff8dfb9 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/C/ee73ca5a49cd4061aa515d3d7ff8dfb9 2024-11-18T20:14:58,816 DEBUG [StoreCloser-TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/C/43f60af7dabd449b84dc21250ad0cfaa to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/C/43f60af7dabd449b84dc21250ad0cfaa 2024-11-18T20:14:58,817 DEBUG [StoreCloser-TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/C/310789de67e842a3933ad57b268d0d80 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/C/310789de67e842a3933ad57b268d0d80 2024-11-18T20:14:58,820 DEBUG [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/recovered.edits/345.seqid, newMaxSeqId=345, maxSeqId=4 2024-11-18T20:14:58,820 INFO [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6. 2024-11-18T20:14:58,820 DEBUG [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.HRegion(1676): Region close journal for 3c05f66f0b4334a1f317161bff1a01e6: Waiting for close lock at 1731960896887Running coprocessor pre-close hooks at 1731960896887Disabling compacts and flushes for region at 1731960896887Disabling writes for close at 1731960897505 (+618 ms)Obtaining lock to block concurrent updates at 1731960897505Preparing flush snapshotting stores in 3c05f66f0b4334a1f317161bff1a01e6 at 1731960897505Finished memstore snapshotting TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6., syncing WAL and waiting on mvcc, flushsize=dataSize=13740, getHeapSize=36720, getOffHeapSize=0, getCellsCount=300 at 1731960897505Flushing stores of TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6. at 1731960897505Flushing 3c05f66f0b4334a1f317161bff1a01e6/A: creating writer at 1731960897505Flushing mob file 3c05f66f0b4334a1f317161bff1a01e6/A: appending metadata at 1731960897509 (+4 ms)Flushing mob file 3c05f66f0b4334a1f317161bff1a01e6/A: closing flushed file at 1731960897509Flushing 3c05f66f0b4334a1f317161bff1a01e6/A: appending metadata at 1731960897919 (+410 ms)Flushing 3c05f66f0b4334a1f317161bff1a01e6/A: closing flushed file at 1731960897920 (+1 ms)Flushing 3c05f66f0b4334a1f317161bff1a01e6/B: creating writer at 1731960898330 (+410 ms)Flushing 3c05f66f0b4334a1f317161bff1a01e6/B: appending metadata at 1731960898332 (+2 ms)Flushing 3c05f66f0b4334a1f317161bff1a01e6/B: closing flushed file at 1731960898332Flushing 3c05f66f0b4334a1f317161bff1a01e6/C: creating writer at 1731960898341 (+9 ms)Flushing 3c05f66f0b4334a1f317161bff1a01e6/C: appending metadata at 1731960898343 (+2 ms)Flushing 3c05f66f0b4334a1f317161bff1a01e6/C: closing flushed file at 1731960898343Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6a976f4b: reopening flushed file at 1731960898755 (+412 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@71c1c262: reopening flushed file at 1731960898761 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@312f5959: reopening flushed file at 1731960898765 (+4 ms)Finished flush of dataSize ~13.42 KB/13740, heapSize ~35.86 KB/36720, currentSize=0 B/0 for 3c05f66f0b4334a1f317161bff1a01e6 in 1263ms, sequenceid=342, compaction requested=true at 1731960898768 (+3 ms)Writing region close event to WAL at 1731960898818 (+50 ms)Running coprocessor post-close hooks at 1731960898820 (+2 ms)Closed at 1731960898820 2024-11-18T20:14:58,822 INFO [RS_CLOSE_REGION-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] handler.UnassignRegionHandler(157): Closed 3c05f66f0b4334a1f317161bff1a01e6 2024-11-18T20:14:58,822 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=170 updating hbase:meta row=3c05f66f0b4334a1f317161bff1a01e6, regionState=CLOSED 2024-11-18T20:14:58,823 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=171, ppid=170, state=RUNNABLE, hasLock=false; CloseRegionProcedure 3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 because future has completed 2024-11-18T20:14:58,825 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=171, resume processing ppid=170 2024-11-18T20:14:58,825 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=171, ppid=170, state=SUCCESS, hasLock=false; CloseRegionProcedure 3c05f66f0b4334a1f317161bff1a01e6, server=c0a89b2656d4,35963,1731960718223 in 2.0900 sec 2024-11-18T20:14:58,827 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=170, resume processing ppid=169 2024-11-18T20:14:58,827 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=170, ppid=169, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=3c05f66f0b4334a1f317161bff1a01e6, UNASSIGN in 2.0940 sec 2024-11-18T20:14:58,828 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=169, resume processing ppid=168 2024-11-18T20:14:58,828 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=169, ppid=168, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 2.0970 sec 2024-11-18T20:14:58,829 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731960898829"}]},"ts":"1731960898829"} 2024-11-18T20:14:58,830 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-18T20:14:58,830 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-18T20:14:58,831 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=168, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 2.1050 sec 2024-11-18T20:14:58,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-18T20:14:58,873 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-18T20:14:58,874 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41277 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-18T20:14:58,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41277 {}] procedure2.ProcedureExecutor(1139): Stored pid=172, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-18T20:14:58,876 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=172, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-18T20:14:58,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=172 2024-11-18T20:14:58,876 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=172, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-18T20:14:58,878 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6 2024-11-18T20:14:58,880 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/A, FileablePath, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/B, FileablePath, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/C, FileablePath, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/recovered.edits] 2024-11-18T20:14:58,883 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/A/7611b6074be54f169d101bac5159f863 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/A/7611b6074be54f169d101bac5159f863 2024-11-18T20:14:58,884 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/A/aafdcab304b44c0f81be4a529a01bd96 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/A/aafdcab304b44c0f81be4a529a01bd96 2024-11-18T20:14:58,884 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/A/d520d1f3249443188151a69ed8d51f62 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/A/d520d1f3249443188151a69ed8d51f62 2024-11-18T20:14:58,886 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/B/5a965c51b4254c91bbf67acc26f0e269 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/B/5a965c51b4254c91bbf67acc26f0e269 2024-11-18T20:14:58,887 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/B/a1c59ea701d14cf7ab841de1e776b2a3 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/B/a1c59ea701d14cf7ab841de1e776b2a3 2024-11-18T20:14:58,887 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/B/c88191c21b844899b8a17fcc37461cf3 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/B/c88191c21b844899b8a17fcc37461cf3 2024-11-18T20:14:58,889 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/C/8e6dceeaa94f446cbe35060ef80d78b8 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/C/8e6dceeaa94f446cbe35060ef80d78b8 2024-11-18T20:14:58,890 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/C/e1f1fa7bc2f146dfb3adf82072ea4dff to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/C/e1f1fa7bc2f146dfb3adf82072ea4dff 2024-11-18T20:14:58,890 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/C/ef6fe427162c4a88a997bf2e018c6e4f to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/C/ef6fe427162c4a88a997bf2e018c6e4f 2024-11-18T20:14:58,892 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/recovered.edits/345.seqid to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6/recovered.edits/345.seqid 2024-11-18T20:14:58,892 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/default/TestAcidGuarantees/3c05f66f0b4334a1f317161bff1a01e6 2024-11-18T20:14:58,893 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-18T20:14:58,893 DEBUG [PEWorker-5 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-18T20:14:58,893 DEBUG [PEWorker-5 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-18T20:14:58,895 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111801e55b058ebd4968b197873aca5a74c9_3c05f66f0b4334a1f317161bff1a01e6 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111801e55b058ebd4968b197873aca5a74c9_3c05f66f0b4334a1f317161bff1a01e6 2024-11-18T20:14:58,896 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411181695edeb131a4f6c967fa05ab4035167_3c05f66f0b4334a1f317161bff1a01e6 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411181695edeb131a4f6c967fa05ab4035167_3c05f66f0b4334a1f317161bff1a01e6 2024-11-18T20:14:58,897 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411181e743e72b3604e3dbbaae4b4cdf72be6_3c05f66f0b4334a1f317161bff1a01e6 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411181e743e72b3604e3dbbaae4b4cdf72be6_3c05f66f0b4334a1f317161bff1a01e6 2024-11-18T20:14:58,898 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411182aeeea248d4c4403a6b917056ac2830b_3c05f66f0b4334a1f317161bff1a01e6 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411182aeeea248d4c4403a6b917056ac2830b_3c05f66f0b4334a1f317161bff1a01e6 2024-11-18T20:14:58,899 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111859a165163e0749388b803fffbdcb6a8d_3c05f66f0b4334a1f317161bff1a01e6 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111859a165163e0749388b803fffbdcb6a8d_3c05f66f0b4334a1f317161bff1a01e6 2024-11-18T20:14:58,899 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111862004b55c5ad445e94e2e92c6a2988a1_3c05f66f0b4334a1f317161bff1a01e6 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111862004b55c5ad445e94e2e92c6a2988a1_3c05f66f0b4334a1f317161bff1a01e6 2024-11-18T20:14:58,900 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411187c649b997f2a45968d92ef991ae57b60_3c05f66f0b4334a1f317161bff1a01e6 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411187c649b997f2a45968d92ef991ae57b60_3c05f66f0b4334a1f317161bff1a01e6 2024-11-18T20:14:58,901 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411187fba2659cef243a2ac0f51b9c073d8e2_3c05f66f0b4334a1f317161bff1a01e6 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411187fba2659cef243a2ac0f51b9c073d8e2_3c05f66f0b4334a1f317161bff1a01e6 2024-11-18T20:14:58,902 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411188a545dd1f1aa48df9a4151cdbc5c128a_3c05f66f0b4334a1f317161bff1a01e6 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411188a545dd1f1aa48df9a4151cdbc5c128a_3c05f66f0b4334a1f317161bff1a01e6 2024-11-18T20:14:58,903 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411188b71096f664f4c1f96722ec22d4dd256_3c05f66f0b4334a1f317161bff1a01e6 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411188b71096f664f4c1f96722ec22d4dd256_3c05f66f0b4334a1f317161bff1a01e6 2024-11-18T20:14:58,903 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111890c35c326e3a48eab86c0b0076ca184e_3c05f66f0b4334a1f317161bff1a01e6 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111890c35c326e3a48eab86c0b0076ca184e_3c05f66f0b4334a1f317161bff1a01e6 2024-11-18T20:14:58,904 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111891224a0705e44ecc9272b377bcc90411_3c05f66f0b4334a1f317161bff1a01e6 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111891224a0705e44ecc9272b377bcc90411_3c05f66f0b4334a1f317161bff1a01e6 2024-11-18T20:14:58,905 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111892e9dea7cc61452e950d628ab3abb045_3c05f66f0b4334a1f317161bff1a01e6 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111892e9dea7cc61452e950d628ab3abb045_3c05f66f0b4334a1f317161bff1a01e6 2024-11-18T20:14:58,906 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111899c5cf6640ad4075931c1029ea8feacd_3c05f66f0b4334a1f317161bff1a01e6 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111899c5cf6640ad4075931c1029ea8feacd_3c05f66f0b4334a1f317161bff1a01e6 2024-11-18T20:14:58,906 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118b77fc8366aca4ea5a782f9c5eebd0069_3c05f66f0b4334a1f317161bff1a01e6 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118b77fc8366aca4ea5a782f9c5eebd0069_3c05f66f0b4334a1f317161bff1a01e6 2024-11-18T20:14:58,907 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118cb4c0c1af09b48f1853504265975fc96_3c05f66f0b4334a1f317161bff1a01e6 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118cb4c0c1af09b48f1853504265975fc96_3c05f66f0b4334a1f317161bff1a01e6 2024-11-18T20:14:58,907 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118d8d21a47b48a4fb2884f76852e1dcec3_3c05f66f0b4334a1f317161bff1a01e6 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118d8d21a47b48a4fb2884f76852e1dcec3_3c05f66f0b4334a1f317161bff1a01e6 2024-11-18T20:14:58,908 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118fbeb55ef774f4095a7744c0f0e6f4279_3c05f66f0b4334a1f317161bff1a01e6 to hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118fbeb55ef774f4095a7744c0f0e6f4279_3c05f66f0b4334a1f317161bff1a01e6 2024-11-18T20:14:58,908 DEBUG [PEWorker-5 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-18T20:14:58,910 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=172, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-18T20:14:58,912 WARN [PEWorker-5 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-18T20:14:58,913 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-18T20:14:58,914 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=172, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-18T20:14:58,914 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-18T20:14:58,914 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731960898914"}]},"ts":"9223372036854775807"} 2024-11-18T20:14:58,916 INFO [PEWorker-5 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-18T20:14:58,916 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 3c05f66f0b4334a1f317161bff1a01e6, NAME => 'TestAcidGuarantees,,1731960870791.3c05f66f0b4334a1f317161bff1a01e6.', STARTKEY => '', ENDKEY => ''}] 2024-11-18T20:14:58,916 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-18T20:14:58,916 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731960898916"}]},"ts":"9223372036854775807"} 2024-11-18T20:14:58,917 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-18T20:14:58,918 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(133): Finished pid=172, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-18T20:14:58,919 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=172, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 44 msec 2024-11-18T20:14:58,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41277 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=172 2024-11-18T20:14:58,983 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-18T20:14:58,984 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-18T20:14:58,991 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testMobGetAtomicity Thread=240 (was 237) - Thread LEAK? -, OpenFileDescriptor=457 (was 449) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=392 (was 443), ProcessCount=11 (was 11), AvailableMemoryMB=2382 (was 2446) 2024-11-18T20:14:58,991 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-18T20:14:58,991 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-18T20:14:58,991 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-18T20:14:58,991 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:14:58,991 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:14:58,991 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T20:14:58,991 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-18T20:14:58,992 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=900566320, stopped=false 2024-11-18T20:14:58,992 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=c0a89b2656d4,41277,1731960717534 2024-11-18T20:14:58,993 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41277-0x100547699a60000, quorum=127.0.0.1:55235, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-18T20:14:58,993 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35963-0x100547699a60001, quorum=127.0.0.1:55235, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-18T20:14:58,993 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41277-0x100547699a60000, quorum=127.0.0.1:55235, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T20:14:58,993 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35963-0x100547699a60001, quorum=127.0.0.1:55235, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T20:14:58,993 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-18T20:14:58,994 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-18T20:14:58,994 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-18T20:14:58,994 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:14:58,994 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'c0a89b2656d4,35963,1731960718223' ***** 2024-11-18T20:14:58,994 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-18T20:14:58,994 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:35963-0x100547699a60001, quorum=127.0.0.1:55235, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-18T20:14:58,994 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:41277-0x100547699a60000, quorum=127.0.0.1:55235, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-18T20:14:58,995 INFO [RS:0;c0a89b2656d4:35963 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-18T20:14:58,995 INFO [RS:0;c0a89b2656d4:35963 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-18T20:14:58,995 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-18T20:14:58,995 INFO [RS:0;c0a89b2656d4:35963 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-18T20:14:58,995 INFO [RS:0;c0a89b2656d4:35963 {}] regionserver.HRegionServer(959): stopping server c0a89b2656d4,35963,1731960718223 2024-11-18T20:14:58,995 INFO [RS:0;c0a89b2656d4:35963 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-18T20:14:58,995 INFO [RS:0;c0a89b2656d4:35963 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;c0a89b2656d4:35963. 2024-11-18T20:14:58,995 DEBUG [RS:0;c0a89b2656d4:35963 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-18T20:14:58,995 DEBUG [RS:0;c0a89b2656d4:35963 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:14:58,995 INFO [RS:0;c0a89b2656d4:35963 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-18T20:14:58,995 INFO [RS:0;c0a89b2656d4:35963 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-18T20:14:58,995 INFO [RS:0;c0a89b2656d4:35963 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-18T20:14:58,995 INFO [RS:0;c0a89b2656d4:35963 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-18T20:14:58,996 INFO [RS:0;c0a89b2656d4:35963 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-18T20:14:58,996 DEBUG [RS:0;c0a89b2656d4:35963 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-11-18T20:14:58,996 DEBUG [RS:0;c0a89b2656d4:35963 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-18T20:14:58,996 DEBUG [RS_CLOSE_META-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-18T20:14:58,996 INFO [RS_CLOSE_META-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-18T20:14:58,996 DEBUG [RS_CLOSE_META-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-18T20:14:58,996 DEBUG [RS_CLOSE_META-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-18T20:14:58,996 DEBUG [RS_CLOSE_META-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-18T20:14:58,996 INFO [RS_CLOSE_META-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=19.91 KB heapSize=35.20 KB 2024-11-18T20:14:59,013 INFO [regionserver/c0a89b2656d4:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-18T20:14:59,018 DEBUG [RS_CLOSE_META-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/hbase/meta/1588230740/.tmp/info/e02a4c27ee514a52bed1d7a0dbf2fe3b is 95, key is TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73./info:/1731960747303/DeleteFamily/seqid=0 2024-11-18T20:14:59,020 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742486_1662 (size=6514) 2024-11-18T20:14:59,196 DEBUG [RS:0;c0a89b2656d4:35963 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-18T20:14:59,396 DEBUG [RS:0;c0a89b2656d4:35963 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-18T20:14:59,422 INFO [RS_CLOSE_META-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=16.75 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/hbase/meta/1588230740/.tmp/info/e02a4c27ee514a52bed1d7a0dbf2fe3b 2024-11-18T20:14:59,445 DEBUG [RS_CLOSE_META-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/hbase/meta/1588230740/.tmp/ns/a9c4b855ae834e24a02b734a9652153e is 93, key is TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73./ns:/1731960747262/DeleteFamily/seqid=0 2024-11-18T20:14:59,448 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742487_1663 (size=5906) 2024-11-18T20:14:59,597 DEBUG [RS:0;c0a89b2656d4:35963 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-18T20:14:59,797 DEBUG [RS:0;c0a89b2656d4:35963 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-18T20:14:59,832 INFO [regionserver/c0a89b2656d4:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-18T20:14:59,832 INFO [regionserver/c0a89b2656d4:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-18T20:14:59,849 INFO [RS_CLOSE_META-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=608 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/hbase/meta/1588230740/.tmp/ns/a9c4b855ae834e24a02b734a9652153e 2024-11-18T20:14:59,874 DEBUG [RS_CLOSE_META-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/hbase/meta/1588230740/.tmp/rep_barrier/5ad14edafbf744e4ad42f6152a8534fb is 102, key is TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73./rep_barrier:/1731960747262/DeleteFamily/seqid=0 2024-11-18T20:14:59,877 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742488_1664 (size=6025) 2024-11-18T20:14:59,997 DEBUG [RS:0;c0a89b2656d4:35963 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-18T20:15:00,197 DEBUG [RS:0;c0a89b2656d4:35963 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-18T20:15:00,278 INFO [RS_CLOSE_META-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=588 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/hbase/meta/1588230740/.tmp/rep_barrier/5ad14edafbf744e4ad42f6152a8534fb 2024-11-18T20:15:00,306 DEBUG [RS_CLOSE_META-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/hbase/meta/1588230740/.tmp/table/0e838e352c4f4ccbbaa39dfe29645bd9 is 96, key is TestAcidGuarantees,,1731960721732.326884d1d40acf488e8bd50aea838c73./table:/1731960747262/DeleteFamily/seqid=0 2024-11-18T20:15:00,310 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742489_1665 (size=5936) 2024-11-18T20:15:00,398 DEBUG [RS:0;c0a89b2656d4:35963 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-18T20:15:00,598 DEBUG [RS:0;c0a89b2656d4:35963 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-18T20:15:00,711 INFO [RS_CLOSE_META-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.99 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/hbase/meta/1588230740/.tmp/table/0e838e352c4f4ccbbaa39dfe29645bd9 2024-11-18T20:15:00,720 DEBUG [RS_CLOSE_META-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/hbase/meta/1588230740/.tmp/info/e02a4c27ee514a52bed1d7a0dbf2fe3b as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/hbase/meta/1588230740/info/e02a4c27ee514a52bed1d7a0dbf2fe3b 2024-11-18T20:15:00,726 INFO [RS_CLOSE_META-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/hbase/meta/1588230740/info/e02a4c27ee514a52bed1d7a0dbf2fe3b, entries=12, sequenceid=90, filesize=6.4 K 2024-11-18T20:15:00,727 DEBUG [RS_CLOSE_META-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/hbase/meta/1588230740/.tmp/ns/a9c4b855ae834e24a02b734a9652153e as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/hbase/meta/1588230740/ns/a9c4b855ae834e24a02b734a9652153e 2024-11-18T20:15:00,732 INFO [RS_CLOSE_META-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/hbase/meta/1588230740/ns/a9c4b855ae834e24a02b734a9652153e, entries=8, sequenceid=90, filesize=5.8 K 2024-11-18T20:15:00,733 DEBUG [RS_CLOSE_META-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/hbase/meta/1588230740/.tmp/rep_barrier/5ad14edafbf744e4ad42f6152a8534fb as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/hbase/meta/1588230740/rep_barrier/5ad14edafbf744e4ad42f6152a8534fb 2024-11-18T20:15:00,737 INFO [RS_CLOSE_META-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/hbase/meta/1588230740/rep_barrier/5ad14edafbf744e4ad42f6152a8534fb, entries=6, sequenceid=90, filesize=5.9 K 2024-11-18T20:15:00,738 DEBUG [RS_CLOSE_META-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/hbase/meta/1588230740/.tmp/table/0e838e352c4f4ccbbaa39dfe29645bd9 as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/hbase/meta/1588230740/table/0e838e352c4f4ccbbaa39dfe29645bd9 2024-11-18T20:15:00,743 INFO [RS_CLOSE_META-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/hbase/meta/1588230740/table/0e838e352c4f4ccbbaa39dfe29645bd9, entries=7, sequenceid=90, filesize=5.8 K 2024-11-18T20:15:00,744 INFO [RS_CLOSE_META-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1748ms, sequenceid=90, compaction requested=false 2024-11-18T20:15:00,750 DEBUG [RS_CLOSE_META-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/data/hbase/meta/1588230740/recovered.edits/93.seqid, newMaxSeqId=93, maxSeqId=1 2024-11-18T20:15:00,751 DEBUG [RS_CLOSE_META-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-18T20:15:00,751 INFO [RS_CLOSE_META-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-18T20:15:00,751 DEBUG [RS_CLOSE_META-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731960898996Running coprocessor pre-close hooks at 1731960898996Disabling compacts and flushes for region at 1731960898996Disabling writes for close at 1731960898996Obtaining lock to block concurrent updates at 1731960898996Preparing flush snapshotting stores in 1588230740 at 1731960898996Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=20384, getHeapSize=35984, getOffHeapSize=0, getCellsCount=191 at 1731960898996Flushing stores of hbase:meta,,1.1588230740 at 1731960898997 (+1 ms)Flushing 1588230740/info: creating writer at 1731960898997Flushing 1588230740/info: appending metadata at 1731960899017 (+20 ms)Flushing 1588230740/info: closing flushed file at 1731960899017Flushing 1588230740/ns: creating writer at 1731960899426 (+409 ms)Flushing 1588230740/ns: appending metadata at 1731960899445 (+19 ms)Flushing 1588230740/ns: closing flushed file at 1731960899445Flushing 1588230740/rep_barrier: creating writer at 1731960899854 (+409 ms)Flushing 1588230740/rep_barrier: appending metadata at 1731960899874 (+20 ms)Flushing 1588230740/rep_barrier: closing flushed file at 1731960899874Flushing 1588230740/table: creating writer at 1731960900286 (+412 ms)Flushing 1588230740/table: appending metadata at 1731960900306 (+20 ms)Flushing 1588230740/table: closing flushed file at 1731960900306Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7fa99d96: reopening flushed file at 1731960900718 (+412 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7731d314: reopening flushed file at 1731960900726 (+8 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1efef5f1: reopening flushed file at 1731960900732 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@21d30795: reopening flushed file at 1731960900737 (+5 ms)Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1748ms, sequenceid=90, compaction requested=false at 1731960900744 (+7 ms)Writing region close event to WAL at 1731960900746 (+2 ms)Running coprocessor post-close hooks at 1731960900750 (+4 ms)Closed at 1731960900751 (+1 ms) 2024-11-18T20:15:00,751 DEBUG [RS_CLOSE_META-regionserver/c0a89b2656d4:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-18T20:15:00,798 INFO [RS:0;c0a89b2656d4:35963 {}] regionserver.HRegionServer(976): stopping server c0a89b2656d4,35963,1731960718223; all regions closed. 2024-11-18T20:15:00,805 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073741834_1010 (size=25344) 2024-11-18T20:15:00,809 DEBUG [RS:0;c0a89b2656d4:35963 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/oldWALs 2024-11-18T20:15:00,809 INFO [RS:0;c0a89b2656d4:35963 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL c0a89b2656d4%2C35963%2C1731960718223.meta:.meta(num 1731960721326) 2024-11-18T20:15:00,813 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073741833_1009 (size=14864986) 2024-11-18T20:15:00,816 DEBUG [RS:0;c0a89b2656d4:35963 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/oldWALs 2024-11-18T20:15:00,816 INFO [RS:0;c0a89b2656d4:35963 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL c0a89b2656d4%2C35963%2C1731960718223:(num 1731960720955) 2024-11-18T20:15:00,816 DEBUG [RS:0;c0a89b2656d4:35963 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T20:15:00,816 INFO [RS:0;c0a89b2656d4:35963 {}] regionserver.LeaseManager(133): Closed leases 2024-11-18T20:15:00,816 INFO [RS:0;c0a89b2656d4:35963 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-18T20:15:00,817 INFO [RS:0;c0a89b2656d4:35963 {}] hbase.ChoreService(370): Chore service for: regionserver/c0a89b2656d4:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-11-18T20:15:00,817 INFO [RS:0;c0a89b2656d4:35963 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-18T20:15:00,817 INFO [regionserver/c0a89b2656d4:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-18T20:15:00,818 INFO [RS:0;c0a89b2656d4:35963 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:35963 2024-11-18T20:15:00,820 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35963-0x100547699a60001, quorum=127.0.0.1:55235, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/c0a89b2656d4,35963,1731960718223 2024-11-18T20:15:00,820 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41277-0x100547699a60000, quorum=127.0.0.1:55235, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-18T20:15:00,820 INFO [RS:0;c0a89b2656d4:35963 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-18T20:15:00,821 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [c0a89b2656d4,35963,1731960718223] 2024-11-18T20:15:00,822 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/c0a89b2656d4,35963,1731960718223 already deleted, retry=false 2024-11-18T20:15:00,822 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; c0a89b2656d4,35963,1731960718223 expired; onlineServers=0 2024-11-18T20:15:00,822 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master 'c0a89b2656d4,41277,1731960717534' ***** 2024-11-18T20:15:00,822 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-18T20:15:00,822 INFO [M:0;c0a89b2656d4:41277 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-18T20:15:00,822 INFO [M:0;c0a89b2656d4:41277 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-18T20:15:00,823 DEBUG [M:0;c0a89b2656d4:41277 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-18T20:15:00,823 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-18T20:15:00,823 DEBUG [M:0;c0a89b2656d4:41277 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-18T20:15:00,823 DEBUG [master/c0a89b2656d4:0:becomeActiveMaster-HFileCleaner.small.0-1731960720583 {}] cleaner.HFileCleaner(306): Exit Thread[master/c0a89b2656d4:0:becomeActiveMaster-HFileCleaner.small.0-1731960720583,5,FailOnTimeoutGroup] 2024-11-18T20:15:00,823 DEBUG [master/c0a89b2656d4:0:becomeActiveMaster-HFileCleaner.large.0-1731960720582 {}] cleaner.HFileCleaner(306): Exit Thread[master/c0a89b2656d4:0:becomeActiveMaster-HFileCleaner.large.0-1731960720582,5,FailOnTimeoutGroup] 2024-11-18T20:15:00,823 INFO [M:0;c0a89b2656d4:41277 {}] hbase.ChoreService(370): Chore service for: master/c0a89b2656d4:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-18T20:15:00,823 INFO [M:0;c0a89b2656d4:41277 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-18T20:15:00,823 DEBUG [M:0;c0a89b2656d4:41277 {}] master.HMaster(1795): Stopping service threads 2024-11-18T20:15:00,823 INFO [M:0;c0a89b2656d4:41277 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-18T20:15:00,823 INFO [M:0;c0a89b2656d4:41277 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-18T20:15:00,823 ERROR [M:0;c0a89b2656d4:41277 {}] procedure2.ProcedureExecutor(763): There are still active thread in group java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10], see STDOUT java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10] Thread[HFileArchiver-6,5,PEWorkerGroup] 2024-11-18T20:15:00,824 INFO [M:0;c0a89b2656d4:41277 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-18T20:15:00,824 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41277-0x100547699a60000, quorum=127.0.0.1:55235, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-18T20:15:00,824 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41277-0x100547699a60000, quorum=127.0.0.1:55235, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T20:15:00,824 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-18T20:15:00,824 DEBUG [M:0;c0a89b2656d4:41277 {}] zookeeper.ZKUtil(347): master:41277-0x100547699a60000, quorum=127.0.0.1:55235, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-18T20:15:00,824 WARN [M:0;c0a89b2656d4:41277 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-18T20:15:00,825 INFO [M:0;c0a89b2656d4:41277 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/.lastflushedseqids 2024-11-18T20:15:00,832 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742490_1666 (size=72) 2024-11-18T20:15:00,923 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35963-0x100547699a60001, quorum=127.0.0.1:55235, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-18T20:15:00,923 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35963-0x100547699a60001, quorum=127.0.0.1:55235, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-18T20:15:00,923 INFO [RS:0;c0a89b2656d4:35963 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-18T20:15:00,924 INFO [RS:0;c0a89b2656d4:35963 {}] regionserver.HRegionServer(1031): Exiting; stopping=c0a89b2656d4,35963,1731960718223; zookeeper connection closed. 2024-11-18T20:15:00,924 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@d091c9 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@d091c9 2024-11-18T20:15:00,925 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-11-18T20:15:01,234 INFO [M:0;c0a89b2656d4:41277 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-18T20:15:01,234 INFO [M:0;c0a89b2656d4:41277 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-18T20:15:01,234 DEBUG [M:0;c0a89b2656d4:41277 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-18T20:15:01,234 INFO [M:0;c0a89b2656d4:41277 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-18T20:15:01,234 DEBUG [M:0;c0a89b2656d4:41277 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-18T20:15:01,234 DEBUG [M:0;c0a89b2656d4:41277 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-18T20:15:01,234 DEBUG [M:0;c0a89b2656d4:41277 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-18T20:15:01,235 INFO [M:0;c0a89b2656d4:41277 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=761.52 KB heapSize=935.91 KB 2024-11-18T20:15:01,257 DEBUG [M:0;c0a89b2656d4:41277 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/f8fc485750c3430cb343458bf87a3502 is 82, key is hbase:meta,,1/info:regioninfo/1731960721413/Put/seqid=0 2024-11-18T20:15:01,260 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742491_1667 (size=5672) 2024-11-18T20:15:01,661 INFO [M:0;c0a89b2656d4:41277 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=2151 (bloomFilter=true), to=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/f8fc485750c3430cb343458bf87a3502 2024-11-18T20:15:01,695 DEBUG [M:0;c0a89b2656d4:41277 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/882d0dcaec0a4c48bf0eda0d3bad180b is 2275, key is \x00\x00\x00\x00\x00\x00\x00\x95/proc:d/1731960873900/Put/seqid=0 2024-11-18T20:15:01,698 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742492_1668 (size=44484) 2024-11-18T20:15:02,100 INFO [M:0;c0a89b2656d4:41277 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=760.96 KB at sequenceid=2151 (bloomFilter=true), to=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/882d0dcaec0a4c48bf0eda0d3bad180b 2024-11-18T20:15:02,110 INFO [M:0;c0a89b2656d4:41277 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 882d0dcaec0a4c48bf0eda0d3bad180b 2024-11-18T20:15:02,129 DEBUG [M:0;c0a89b2656d4:41277 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/d7fe3f2cc4764e0eb28aed74eae665ca is 69, key is c0a89b2656d4,35963,1731960718223/rs:state/1731960720725/Put/seqid=0 2024-11-18T20:15:02,132 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073742493_1669 (size=5156) 2024-11-18T20:15:02,534 INFO [M:0;c0a89b2656d4:41277 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=2151 (bloomFilter=true), to=hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/d7fe3f2cc4764e0eb28aed74eae665ca 2024-11-18T20:15:02,543 DEBUG [M:0;c0a89b2656d4:41277 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/f8fc485750c3430cb343458bf87a3502 as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/f8fc485750c3430cb343458bf87a3502 2024-11-18T20:15:02,547 INFO [M:0;c0a89b2656d4:41277 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/f8fc485750c3430cb343458bf87a3502, entries=8, sequenceid=2151, filesize=5.5 K 2024-11-18T20:15:02,548 DEBUG [M:0;c0a89b2656d4:41277 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/882d0dcaec0a4c48bf0eda0d3bad180b as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/882d0dcaec0a4c48bf0eda0d3bad180b 2024-11-18T20:15:02,551 INFO [M:0;c0a89b2656d4:41277 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 882d0dcaec0a4c48bf0eda0d3bad180b 2024-11-18T20:15:02,551 INFO [M:0;c0a89b2656d4:41277 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/882d0dcaec0a4c48bf0eda0d3bad180b, entries=172, sequenceid=2151, filesize=43.4 K 2024-11-18T20:15:02,552 DEBUG [M:0;c0a89b2656d4:41277 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/d7fe3f2cc4764e0eb28aed74eae665ca as hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/d7fe3f2cc4764e0eb28aed74eae665ca 2024-11-18T20:15:02,555 INFO [M:0;c0a89b2656d4:41277 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44393/user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/d7fe3f2cc4764e0eb28aed74eae665ca, entries=1, sequenceid=2151, filesize=5.0 K 2024-11-18T20:15:02,556 INFO [M:0;c0a89b2656d4:41277 {}] regionserver.HRegion(3140): Finished flush of dataSize ~761.52 KB/779796, heapSize ~935.61 KB/958064, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1322ms, sequenceid=2151, compaction requested=false 2024-11-18T20:15:02,557 INFO [M:0;c0a89b2656d4:41277 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-18T20:15:02,557 DEBUG [M:0;c0a89b2656d4:41277 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731960901234Disabling compacts and flushes for region at 1731960901234Disabling writes for close at 1731960901234Obtaining lock to block concurrent updates at 1731960901235 (+1 ms)Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1731960901235Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=779796, getHeapSize=958304, getOffHeapSize=0, getCellsCount=2348 at 1731960901235Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1731960901236 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1731960901236Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1731960901257 (+21 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1731960901257Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1731960901670 (+413 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1731960901695 (+25 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1731960901695Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1731960902111 (+416 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1731960902129 (+18 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1731960902129Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4fe19e67: reopening flushed file at 1731960902542 (+413 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@26c45bd4: reopening flushed file at 1731960902547 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@74156b1f: reopening flushed file at 1731960902551 (+4 ms)Finished flush of dataSize ~761.52 KB/779796, heapSize ~935.61 KB/958064, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1322ms, sequenceid=2151, compaction requested=false at 1731960902556 (+5 ms)Writing region close event to WAL at 1731960902557 (+1 ms)Closed at 1731960902557 2024-11-18T20:15:02,560 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41017 is added to blk_1073741830_1006 (size=920818) 2024-11-18T20:15:02,560 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /user/jenkins/test-data/4fafab01-2074-8acf-df5c-907bc14c6123/MasterData/WALs/c0a89b2656d4,41277,1731960717534/c0a89b2656d4%2C41277%2C1731960717534.1731960719932 not finished, retry = 0 2024-11-18T20:15:02,662 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-18T20:15:02,662 INFO [M:0;c0a89b2656d4:41277 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-18T20:15:02,662 INFO [M:0;c0a89b2656d4:41277 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:41277 2024-11-18T20:15:02,663 INFO [M:0;c0a89b2656d4:41277 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-18T20:15:02,765 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41277-0x100547699a60000, quorum=127.0.0.1:55235, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-18T20:15:02,765 INFO [M:0;c0a89b2656d4:41277 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-18T20:15:02,765 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41277-0x100547699a60000, quorum=127.0.0.1:55235, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-18T20:15:02,771 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@2c68b1bf{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-18T20:15:02,774 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@3efe07f8{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-18T20:15:02,774 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-18T20:15:02,774 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@42946cd0{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-18T20:15:02,775 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@336e9627{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/098aacef-40a4-13ba-ed62-ee93d3ee3293/hadoop.log.dir/,STOPPED} 2024-11-18T20:15:02,778 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-18T20:15:02,779 WARN [BP-138538035-172.17.0.2-1731960714281 heartbeating to localhost/127.0.0.1:44393 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-18T20:15:02,779 WARN [BP-138538035-172.17.0.2-1731960714281 heartbeating to localhost/127.0.0.1:44393 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-138538035-172.17.0.2-1731960714281 (Datanode Uuid f25a728a-67d8-468a-a294-678d073a2b85) service to localhost/127.0.0.1:44393 2024-11-18T20:15:02,779 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-18T20:15:02,780 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/098aacef-40a4-13ba-ed62-ee93d3ee3293/cluster_08cd2a1d-a16d-5ba4-ed14-6adb17e4da61/data/data1/current/BP-138538035-172.17.0.2-1731960714281 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-18T20:15:02,780 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-18T20:15:02,782 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/098aacef-40a4-13ba-ed62-ee93d3ee3293/cluster_08cd2a1d-a16d-5ba4-ed14-6adb17e4da61/data/data2/current/BP-138538035-172.17.0.2-1731960714281 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-18T20:15:02,790 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4ded99d3{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-18T20:15:02,791 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@557285fa{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-18T20:15:02,791 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-18T20:15:02,791 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@93c62df{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-18T20:15:02,792 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6b23711f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/098aacef-40a4-13ba-ed62-ee93d3ee3293/hadoop.log.dir/,STOPPED} 2024-11-18T20:15:02,813 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-18T20:15:02,961 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down